diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json
index 66c25adf..35c9c1eb 100644
--- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json
+++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_adjudications.json
@@ -1,7 +1,7 @@
{
"schema_version": 2,
"note": "Developer adjudications of judge verdicts. An entry either resolves a non-final judge class to a final one (llm_error, parse_contract_failure) or affirms prompt_ambiguity, which is allowed only together with excluded_from_scoring: true, meaning the output is removed from scoring for every model and listed in the run's reference_exclusions.json (the freezer verifies the two records agree). Each entry keeps the judge's verdict verbatim beside the decision and the reasoning. Applied to the row annotations and case notes by scripts/apply_adjudications.py before export.",
- "date_conventions": "adjudicated_on names the audit wave that made the decision (2026-09-05, 2026-09-22, 2026-09-29, 2026-10-05, 2026-10-06, 2026-10-10 or 2026-10-14). It carries no time of day or time zone. The 2026-09-05, 2026-09-22 and 2026-09-29 waves' decisions were written up to the day each wave's release was committed (2026-09-05, 2026-09-23 and 2026-09-30); the 2026-09-29 wave's reference sweep began on 2026-09-29 UTC, before its decisions were written. The 2026-10-05 wave's decisions follow Max's rulings of 2026-10-05 US Eastern time (2026-10-06 UTC) and were written on 2026-10-06 UTC. The 2026-10-06 wave's decisions follow Max's rulings of 2026-10-06 (17:11 UTC) and were written on 2026-10-09 UTC. The 2026-10-10 wave's decisions exclude the outputs the references' engine upgrade moved onto an input the prompt does not state, and were written on 2026-10-10 UTC. The 2026-10-14 wave's decisions keep scored outputs on cases this release's new models re-opened, affirming a reference the case's verdict flagged or resolving rows the judge gave a class outside the final set, and were written on 2026-10-14 UTC. judged_on_utc, judge_rejudged_on and each judge_previous item's judged_on are UTC days, read from the verdict's sidecar (verdict.meta.json judged_at_utc). The top-level judge fields name the case's verdict in this release's audit tree, dated by judge_rejudged_on when a later wave judged the case again and by judged_on_utc otherwise, and judge_previous keeps the verdict that re-judge replaced. A wave's own judge runs could finish after one of its decisions was first written, so a recorded verdict can postdate adjudicated_on up to the last day its wave's decisions were written. A flag an earlier judge run raised stays recorded and names that run in judge_reference_suspect_source, at the top level or on the judge_previous item it belongs to. Where a later wave replaced the verdict a decision reviewed without keeping it, adjudicated_verdict gives that verdict as the decision's own release recorded it.",
+ "date_conventions": "adjudicated_on names the audit wave that made the decision (2026-09-05, 2026-09-22, 2026-09-29, 2026-10-05, 2026-10-06, 2026-10-10, 2026-10-14, 2026-11-01 or 2026-11-11). It carries no time of day or time zone. The 2026-09-05, 2026-09-22 and 2026-09-29 waves' decisions were written up to the day each wave's release was committed (2026-09-05, 2026-09-23 and 2026-09-30); the 2026-09-29 wave's reference sweep began on 2026-09-29 UTC, before its decisions were written. The 2026-10-05 wave's decisions follow Max's rulings of 2026-10-05 US Eastern time (2026-10-06 UTC) and were written on 2026-10-06 UTC. The 2026-10-06 wave's decisions follow Max's rulings of 2026-10-06 (17:11 UTC) and were written on 2026-10-09 UTC. The 2026-10-10 wave's decisions exclude the outputs the references' engine upgrade moved onto an input the prompt does not state, and were written on 2026-10-10 UTC. The 2026-10-14 wave's decisions keep scored outputs on cases this release's new models re-opened, affirming a reference the case's verdict flagged or resolving rows the judge gave a class outside the final set, and were written on 2026-10-14 UTC. The 2026-11-01 wave's decision follows Max's ruling of 2026-11-01 (21:06 UTC) and was written on 2026-11-11 UTC. The 2026-11-11 wave's decisions keep scored outputs on cases this release's new models re-opened, affirming a reference the case's verdict flagged or resolving rows the judge gave a class outside the final set, and were written on 2026-11-11 UTC. judged_on_utc, judge_rejudged_on and each judge_previous item's judged_on are UTC days, read from the verdict's sidecar (verdict.meta.json judged_at_utc). The top-level judge fields name the case's verdict in this release's audit tree, dated by judge_rejudged_on when a later wave judged the case again and by judged_on_utc otherwise, and judge_previous keeps the verdict that re-judge replaced. A wave's own judge runs could finish after one of its decisions was first written, so a recorded verdict can postdate adjudicated_on up to the last day its wave's decisions were written. A flag an earlier judge run raised stays recorded and names that run in judge_reference_suspect_source, at the top level or on the judge_previous item it belongs to. Where a later wave replaced the verdict a decision reviewed without keeping it, adjudicated_verdict gives that verdict as the decision's own release recorded it.",
"adjudications": [
{
"country": "us",
@@ -269,7 +269,7 @@
"scenario_id": "scenario_023",
"variable": "snap",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "age_disability",
"adjudicated_failure_source": "prompt_ambiguity",
@@ -280,7 +280,7 @@
"adjudicator": "developer",
"reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 3,576.00 with every publication convention and upstream fix (the frozen-engine value above was 3,596.04).",
"judge_reference_suspect": false,
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -323,6 +323,13 @@
"judge_failure_subtype": "age_disability",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "age_disability",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
],
"adjudicated_verdict": {
@@ -338,7 +345,7 @@
"scenario_id": "scenario_057",
"variable": "snap",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
"judge_failure_source": "reference_model_issue_fixed",
"judge_failure_subtype": "household_unit_or_filing_status",
"adjudicated_failure_source": "prompt_ambiguity",
@@ -351,7 +358,7 @@
"judge_reference_suspect": true,
"reference_verdict": "unlisted_input",
"reference_basis": "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -397,6 +404,13 @@
"judge_reference_suspect": true,
"judged_on": "2026-10-14",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "reference_model_issue_fixed",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27"
}
],
"adjudicated_verdict": {
@@ -412,7 +426,7 @@
"scenario_id": "scenario_100",
"variable": "snap",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "taxable_income_or_deductions",
"adjudicated_failure_source": "prompt_ambiguity",
@@ -423,7 +437,7 @@
"adjudicator": "developer",
"reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, since fixed upstream by PolicyEngine/policyengine-us#10033, which the policyengine-us 2.41.0 references include) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification.",
"judge_reference_suspect": false,
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -466,6 +480,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
],
"adjudicated_verdict": {
@@ -481,7 +502,7 @@
"scenario_id": "scenario_057",
"variable": "ssi",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
"judge_failure_source": "llm_error",
"judge_failure_subtype": "categorical_eligibility",
"adjudicated_failure_source": "prompt_ambiguity",
@@ -495,7 +516,7 @@
"reference_verdict": "unlisted_input",
"reference_basis": "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -542,6 +563,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-10-14",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "categorical_eligibility",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
],
"adjudicated_verdict": {
@@ -557,9 +586,9 @@
"scenario_id": "scenario_064",
"variable": "ssi",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
"judge_failure_source": "prompt_ambiguity",
- "judge_failure_subtype": "age_disability",
+ "judge_failure_subtype": "categorical_eligibility",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "age_disability",
"excluded_from_scoring": true,
@@ -568,7 +597,7 @@
"adjudicator": "developer",
"reasoning": "The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 11928.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading, fourteen of the eighteen models that paid SSI on the 51-model board of 2026-10-27 land within 10% of the $11,928 federal benefit (ten exactly); the other four applied an in-kind support reduction the facts do not support: one-third for deepseek-v4-pro, deepseek-v4-pro-0813 and inkling, and $350.33 a month for deepseek-v4.1-flash.",
"judge_reference_suspect": false,
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -590,6 +619,13 @@
"judge_failure_subtype": "age_disability",
"judge_reference_suspect": false,
"judged_on": "2026-09-30"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "prompt_ambiguity",
+ "judge_failure_subtype": "age_disability",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
],
"adjudicated_verdict": {
@@ -605,8 +641,8 @@
"scenario_id": "scenario_067",
"variable": "ssi",
"judge_model": "claude-opus-5-5",
- "judged_on_utc": "2026-10-27",
- "judge_failure_source": "reference_data_issue_fixed",
+ "judged_on_utc": "2026-11-11",
+ "judge_failure_source": "reference_engine_defect",
"judge_failure_subtype": "categorical_eligibility",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "age_disability",
@@ -618,7 +654,7 @@
"judge_reference_suspect": true,
"reference_verdict": "unlisted_input",
"reference_basis": "42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -647,6 +683,13 @@
"judge_failure_subtype": "categorical_eligibility",
"judge_reference_suspect": true,
"judged_on": "2026-10-13"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "reference_data_issue_fixed",
+ "judge_failure_subtype": "categorical_eligibility",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27"
}
],
"adjudicated_verdict": {
@@ -701,7 +744,7 @@
"reference_verdict": "engine_defect",
"reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
"reasoning": "The frozen reference is 106,505.90; the recorded correction is 107,198.34 under the refund-excluded reading. The frozen engine omitted the IRA active-participant phase-out. The policyengine-us 2.60.0 engine applies that phase-out and includes the refund, giving 108,525.58. The output remains excluded because refund taxation, excess-deferral allocation and NIIT inclusion remain unresolved.",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -744,6 +787,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27"
}
]
},
@@ -764,7 +814,7 @@
"reference_basis": "whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say",
"reasoning": "The frozen reference is 68,056.71; the alternative is 68,112.09. The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it.",
"judge_reference_suspect_source": "claude-opus-5-5 judge run adjudicated 2026-09-22",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -812,6 +862,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-10-14",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -882,7 +940,7 @@
"scenario_id": "scenario_056",
"variable": "snap",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_engine_defect",
+ "judge_failure_source": "llm_error",
"judge_failure_subtype": "taxable_income_or_deductions",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "categorical_eligibility",
@@ -893,7 +951,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies",
"reasoning": "The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.",
- "judge_rejudged_on": "2026-10-27",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -939,6 +998,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "reference_engine_defect",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27"
}
]
},
@@ -1260,8 +1326,8 @@
"scenario_id": "scenario_108",
"variable": "state_refundable_credits",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_model_issue_fixed",
- "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "state_local_rule",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "state_local_rule",
"adjudicated_on": "2026-09-22",
@@ -1271,7 +1337,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits",
"reasoning": "The frozen reference is 271.39; the alternative is 0.00. The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out.",
- "judge_rejudged_on": "2026-10-27",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1318,6 +1385,13 @@
"judge_reference_suspect": true,
"judged_on": "2026-10-14",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "reference_model_issue_fixed",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27"
}
]
},
@@ -1326,7 +1400,7 @@
"scenario_id": "scenario_112",
"variable": "snap",
"judge_model": "claude-opus-5-5",
- "judge_failure_source": "reference_engine_defect",
+ "judge_failure_source": "reference_model_issue_fixed",
"judge_failure_subtype": "taxable_income_or_deductions",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "categorical_eligibility",
@@ -1337,7 +1411,7 @@
"reference_verdict": "unlisted_input",
"reference_basis": "weekly_hours_worked_before_lsr",
"reasoning": "The frozen reference is 287.68; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement.",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1384,6 +1458,13 @@
"judge_reference_suspect": true,
"judged_on": "2026-10-14",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "reference_engine_defect",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27"
}
]
},
@@ -1403,7 +1484,7 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the listed home mortgage interest is on the home the SNAP household occupies",
"reasoning": "The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value.",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1446,6 +1527,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -1465,7 +1553,7 @@
"reference_verdict": "engine_defect",
"reference_basis": "26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges)",
"reasoning": "The frozen reference is 40,021.82; the corrected value is 40,028.32. The frozen engine deducted traditional IRA contributions without the active-participant phase-out.",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1508,6 +1596,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -1563,7 +1658,7 @@
"reference_basis": "42 U.S.C. 1397jj(b)(1)(C); 42 CFR 457.310(b)(2)(ii); Pennsylvania CHIP state plan 4.1.7",
"reasoning": "Child 3 has employer-sponsored group coverage, which bars CHIP; the household's income passes Pennsylvania's CHIP limit, so the judge's income-limit hypothesis fails.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-10-13",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1580,6 +1675,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-09-29",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "health_coverage",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-13",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1589,7 +1692,7 @@
"variable": "state_refundable_credits",
"judge_model": "claude-opus-5-5",
"judge_failure_source": "reference_engine_defect",
- "judge_failure_subtype": "state_local_rule",
+ "judge_failure_subtype": "credit_phaseout",
"adjudicated_failure_source": "llm_error",
"adjudicated_failure_subtype": "credit_phaseout",
"adjudicated_on": "2026-09-22",
@@ -1598,7 +1701,7 @@
"reference_verdict": "affirmed",
"reference_basis": "N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32",
"reasoning": "New Jersey bases the earned income credit for filers 18 and older who fail only the federal age test on the federal maximum credit for filers without a qualifying child, and pays it as a flat yearly amount: 40% of the 2026 $664 maximum is $265.60. The judge's phase-in reading fails.",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1641,6 +1744,13 @@
"judge_failure_subtype": "credit_phaseout",
"judge_reference_suspect": true,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "reference_engine_defect",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27"
}
]
},
@@ -1660,7 +1770,7 @@
"reference_basis": "26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12",
"reasoning": "The 2026 educator expense cap is $350, so the head's $337.50 is fully deductible; the judge's $300 cap is not the law.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1698,6 +1808,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-10-14",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1848,7 +1966,7 @@
"reference_basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C)",
"reasoning": "The frozen 287.68 is regenerated as 288.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze): The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published).",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1897,6 +2015,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-10-14",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "categorical_eligibility",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1916,7 +2042,7 @@
"reference_basis": "26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7",
"reasoning": "The frozen 1,979.16 is regenerated as 2,361.96: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -1965,6 +2091,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-10-13",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -1974,7 +2108,7 @@
"variable": "state_income_tax_before_refundable_credits",
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_subtype": "thresholds_rates",
"adjudicated_failure_source": "reference_later_law",
"adjudicated_failure_subtype": "taxable_income_or_deductions",
"adjudicated_on": "2026-10-06",
@@ -1985,7 +2119,7 @@
"reference_basis": "Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published",
"reasoning": "The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. The frozen reference is 820.35; the value under the published figure is 820.26. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana).",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-10-27",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2034,6 +2168,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-10-14",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -2108,7 +2250,7 @@
"reference_basis": "7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 CFR 273.10(e)(2)(ii)(A)",
"reasoning": "The frozen 8,020.55 is regenerated as 7,932.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze): If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment.",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the case's current verdict.json does not flag it",
- "judge_rejudged_on": "2026-10-14",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2149,6 +2291,14 @@
"judge_reference_suspect": true,
"judged_on": "2026-10-13",
"judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-14",
+ "judge_reference_suspect_source": "an earlier judge run in the 2026-09-22 wave (flagged_sept22_wave.json); the verdict dated here does not flag it"
}
]
},
@@ -2168,8 +2318,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)",
"reasoning": "The frozen reference is 3,818.15; the alternative is 3,788.99. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122).",
- "judged_on_utc": "2026-10-13",
- "judge_rejudged_on": "2026-10-13",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2184,6 +2334,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-10-08"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-13"
}
]
},
@@ -2203,8 +2360,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)",
"reasoning": "The frozen reference is 24,164.46; the alternative is 23,772.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122).",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2240,6 +2397,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -2259,8 +2423,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom)",
"reasoning": "The frozen reference is 24,961.34; the alternative is 24,391.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122).",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2296,6 +2460,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -2400,8 +2571,8 @@
"adjudicator": "developer",
"judge_reference_suspect": false,
"reasoning": "The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied New Jersey's pre-2026 child tax credit schedule. The reference applies P.L.2026, c.26, approved June 30, 2026, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors.",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2437,6 +2608,13 @@
"judge_failure_subtype": "state_local_rule",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "reference_later_law",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -2446,7 +2624,7 @@
"variable": "federal_income_tax_before_refundable_credits",
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_subtype": "household_unit_or_filing_status",
"adjudicated_failure_source": "prompt_ambiguity",
"adjudicated_failure_subtype": "taxable_income_or_deductions",
"adjudicated_on": "2026-10-05",
@@ -2456,8 +2634,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set",
"reasoning": "The frozen reference is 11,113.57; the alternative is 11,895.23. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of California withholding on federal AGI ($6,058.85), while read as the household paying its 2026 California income tax during the year the deduction takes the engine's liability ($2,505.87). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py).",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2486,6 +2664,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -2505,8 +2690,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax",
"reasoning": "The frozen reference is 2,346.10; the alternative is 2,218.50. The prompt counts mandatory employee state payroll taxes; Minnesota Paid Leave contributions are the employer's, which may deduct the employee share from wages but need not (Minn. Stat. 268B.14, subds. 1, 3 and 5a), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($127.60); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json).",
- "judged_on_utc": "2026-10-14",
- "judge_rejudged_on": "2026-10-14",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2528,6 +2713,13 @@
"judge_failure_subtype": "state_local_rule",
"judge_reference_suspect": false,
"judged_on": "2026-10-13"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "prompt_ambiguity",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-14"
}
]
},
@@ -2547,8 +2739,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax",
"reasoning": "The frozen reference is 330.73; the alternative is 312.74. The prompt counts mandatory employee state payroll taxes; Colorado FAMLI contributions are the employer's, which may deduct the employee share from wages but need not (C.R.S. 8-13.3-507(2) and (5); 7 CCR 1107-1 s. 1.4(6)(A)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($17.99); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json).",
- "judged_on_utc": "2026-10-13",
- "judge_rejudged_on": "2026-10-13",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2563,6 +2755,13 @@
"judge_failure_subtype": "state_local_rule",
"judge_reference_suspect": false,
"judged_on": "2026-10-08"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "prompt_ambiguity",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-13"
}
]
},
@@ -2582,8 +2781,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set",
"reasoning": "The frozen reference is 26,932.27; the alternative is 26,991.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Massachusetts withholding on federal AGI ($8,484.41), while read as the household paying its 2026 Massachusetts income tax during the year the deduction takes the engine's liability ($8,238.41). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py).",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2612,6 +2811,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -2673,8 +2879,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax",
"reasoning": "The frozen reference is 8,108.03; the alternative is 7,664.92. The prompt counts mandatory employee state payroll taxes; New York disability and Paid Family Leave contributions are the employer's, which may deduct the employee share from wages but need not (N.Y. Workers' Compensation Law ss. 209(3)-(4) and 210(1)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($443.11); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json).",
- "judged_on_utc": "2026-10-14",
- "judge_rejudged_on": "2026-10-14",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2696,6 +2902,13 @@
"judge_failure_subtype": "state_local_rule",
"judge_reference_suspect": false,
"judged_on": "2026-10-13"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "prompt_ambiguity",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-14"
}
]
},
@@ -2715,8 +2928,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set; also Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either",
"reasoning": "The frozen reference is 10,729.61; the alternative is 11,137.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Virginia withholding on federal AGI ($5,367.30), while read as the household paying its 2026 Virginia income tax during the year the deduction takes the engine's liability ($3,514.15). The reference's medical deduction also counts a modeled Medicare Part B premium ($2,434.80) the prompt does not list; without it the output is 11,265.27. Found in the 2026-10-05 audits of the state income tax in SALT (reference_audit/2026-10-05) and of the Part B premium (reference_audit/2026-10-05-medicare-part-b).",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2745,6 +2958,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": true,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -2764,8 +2984,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either",
"reasoning": "The frozen reference is 3,514.15; the alternative is 3,654.15. The prompt lists no Medicare enrollment or Part B premium; policyengine-us 2.15.17 treats the 69-year-old head as enrolled (takes_up_medicare_if_eligible defaults to true) and adds the $2,434.80 standard Part B premium to medical expenses, which Virginia's itemized deductions carry from the federal ones; read with no premium paid, the output is the alternative. Found in the 2026-10-05 audit of the Part B premium (reference_audit/2026-10-05-medicare-part-b, scripts/sweep_part_b.py).",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2794,6 +3014,13 @@
"judge_failure_subtype": "taxable_income_or_deductions",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -2813,8 +3040,8 @@
"reference_verdict": "later_law",
"reference_basis": "Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published",
"reasoning": "The frozen reference is 305.14; the value under the published figure is 305.05. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana).",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2836,6 +3063,13 @@
"judge_failure_subtype": "thresholds_rates",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -2855,8 +3089,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
"reasoning": "The frozen reference is 8,628.99; the alternative is 11,848.99. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2878,6 +3112,13 @@
"judge_failure_subtype": "household_unit_or_filing_status",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -2897,8 +3138,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
"reasoning": "The frozen reference is 3,388.25; the alternative is 4,528.08. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2920,6 +3161,13 @@
"judge_failure_subtype": "household_unit_or_filing_status",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -2939,8 +3187,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
"reasoning": "The frozen reference is 5,200.24; the alternative is 8,420.24. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -2962,6 +3210,13 @@
"judge_failure_subtype": "household_unit_or_filing_status",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -2981,8 +3236,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own",
"reasoning": "The frozen reference is 3,070.06; the alternative is 4,451.56. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2.",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -3004,6 +3259,13 @@
"judge_failure_subtype": "household_unit_or_filing_status",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -3023,8 +3285,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "county of residence (Indiana county income tax)",
"reasoning": "The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state.",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -3046,6 +3308,13 @@
"judge_failure_subtype": "state_local_rule",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "prompt_ambiguity",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -3065,8 +3334,8 @@
"reference_verdict": "unlisted_input",
"reference_basis": "county of residence (Indiana county income tax)",
"reasoning": "The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state.",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -3088,6 +3357,13 @@
"judge_failure_subtype": "state_local_rule",
"judge_reference_suspect": false,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "prompt_ambiguity",
+ "judge_failure_subtype": "state_local_rule",
+ "judge_reference_suspect": false,
+ "judged_on": "2026-10-27"
}
]
},
@@ -3112,7 +3388,7 @@
"variable": "state_income_tax_before_refundable_credits",
"judge_model": "claude-opus-5-5",
"judge_failure_source": "llm_error",
- "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_failure_subtype": "thresholds_rates",
"adjudicated_failure_source": "llm_error",
"adjudicated_failure_subtype": "state_local_rule",
"adjudicated_on": "2026-10-14",
@@ -3121,8 +3397,8 @@
"reference_verdict": "affirmed",
"reference_basis": "Wis. Stat. 71.05(10)(c)2. (for taxable years beginning after December 31, 2022, a net capital loss offsets ordinary income up to $3,000, or $1,500 for married persons filing separately); 2025 Wisconsin Schedule WD, line 28",
"reasoning": "The judge flagged the reference on the hypothesis that Wisconsin limits the net capital loss deduction to $500, and labeled claude-opus-5.5's row reference_model_issue_fixed because it applied that limit. The $500 limit is Wis. Stat. 71.05(10)(c)1., which covers only taxable years beginning before January 1, 2023; for 2026, subd. 2. allows $3,000 on a joint return, the $3,000 the reference deducts, so Wisconsin adds nothing back and the reference's Wisconsin AGI of $114,421 stands. The rows are model errors.",
- "judged_on_utc": "2026-10-27",
- "judge_rejudged_on": "2026-10-27",
+ "judged_on_utc": "2026-11-11",
+ "judge_rejudged_on": "2026-11-11",
"judge_previous": [
{
"judge_model": "claude-opus-5-5",
@@ -3130,8 +3406,80 @@
"judge_failure_subtype": "state_local_rule",
"judge_reference_suspect": true,
"judged_on": "2026-10-14"
+ },
+ {
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "taxable_income_or_deductions",
+ "judge_reference_suspect": true,
+ "judged_on": "2026-10-27"
}
]
+ },
+ {
+ "country": "us",
+ "scenario_id": "scenario_122",
+ "variable": "state_refundable_credits",
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "credit_phaseout",
+ "adjudicated_failure_source": "prompt_ambiguity",
+ "adjudicated_failure_subtype": "state_local_rule",
+ "adjudicated_on": "2026-11-01",
+ "adjudicator": "developer",
+ "excluded_from_scoring": true,
+ "judge_reference_suspect": false,
+ "reference_verdict": "unlisted_input",
+ "reference_basis": "a qualifying Certificate of Rent Paid (mn_renters_credit_qualifying_crp)",
+ "reasoning": "The frozen reference is 0.00; the alternative is 1,921.00. The prompt places the household in Minnesota with pre-subsidy rent of $28,235 and states no Certificate of Rent Paid (CRP); policyengine-us 2.60.0 grants Minnesota's renter's credit only with a qualifying CRP (mn_renters_credit_qualifying_crp, a boolean input that defaults to false), while Minn. Stat. 290.0693 subd. 4 requires the owner or managing agent of a rented homestead to furnish one to a renter on December 31; read as a renter who holds the CRP, the household's renter's credit household income of $65,346.18 falls under 2026's $79,330 first zero threshold, and the credit, and with it the output, is $1,921. The 2026-10-29 audit traced it (reference_audit/2026-10-29-mn-renters-crp), and Max's decision d2180 excludes it.",
+ "judged_on_utc": "2026-10-27"
+ },
+ {
+ "country": "us",
+ "scenario_id": "scenario_039",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "household_unit_or_filing_status",
+ "adjudicated_failure_source": "llm_error",
+ "adjudicated_failure_subtype": "household_unit_or_filing_status",
+ "adjudicated_on": "2026-11-11",
+ "adjudicator": "developer",
+ "judge_reference_suspect": false,
+ "reasoning": "The judge labeled eight models' rows reference_later_law (deepseek-v4-pro, gemini-3-flash-preview, gemini-3.1-pro-preview, gemini-3.5-flash, gemini-4-argon, grok-4.5, grok-4.6 and grok-4.7) because they applied the pre-TCJA rate schedule scheduled to return in 2026, with its 15% second bracket, and in most cases a smaller standard deduction and a personal exemption. The reference applies the rate tables, larger standard deduction and zero exemption amount that P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, made permanent, at the 2026 single amounts of Rev. Proc. 2025-32 (a $16,100 standard deduction and the 10%, 12% and 22% brackets), and the benchmark scores law published before the freeze; the rows are model errors, as the 2026-09-29 and 2026-10-14 waves decided for the same rule in scenario_007's and scenario_009's federal income tax. The case keeps the verdict's class: the widowed head has no dependent child, so 26 U.S.C. 2(a)(1)(B) denies surviving-spouse status and the head files single.",
+ "judged_on_utc": "2026-11-11"
+ },
+ {
+ "country": "us",
+ "scenario_id": "scenario_100",
+ "variable": "state_refundable_credits",
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
+ "adjudicated_failure_source": "llm_error",
+ "adjudicated_failure_subtype": "state_local_rule",
+ "adjudicated_on": "2026-11-11",
+ "adjudicator": "developer",
+ "judge_reference_suspect": true,
+ "reference_verdict": "affirmed",
+ "reference_basis": "Mont. Code Ann. 15-30-2318(2) and (4) (2025), as amended by Ch. 227, Laws of 2025 (HB 337), secs. 3 and 6(1): a refundable credit of 20% of the federal earned income credit for the tax year beginning January 1, 2026; Mont. Code Ann. Title 15, ch. 30, part 23 (2025), which holds no child tax credit; minutes of the Montana House Appropriations Committee, February 23, 2023, which tabled HB 268",
+ "reasoning": "The judge flagged the reference on the hypothesis that Montana's HB 268 (2023) pays a refundable $1,200 credit for each child under 6. HB 268 never became law: the House Appropriations Committee tabled it on February 23, 2023, and it died there. No later act created a Montana child tax credit; Title 15, chapter 30, part 23 of the 2025 Montana Code has none, and the Department of Revenue's 2025 Form 2 instructions name only HB 129 and HB 337 among the changes for tax year 2026. For 2026 the household's one Montana refundable credit is the earned income tax credit, 20% of the federal credit: 40% of $2,055.72 (wages of $5,914.72 less $3,859 of traditional 401(k) deferrals) gives $822.29, and 20% of that gives $164.46, the reference. The rows are model errors.",
+ "judged_on_utc": "2026-11-11"
+ },
+ {
+ "country": "us",
+ "scenario_id": "scenario_101",
+ "variable": "federal_income_tax_before_refundable_credits",
+ "judge_model": "claude-opus-5-5",
+ "judge_failure_source": "llm_error",
+ "judge_failure_subtype": "thresholds_rates",
+ "adjudicated_failure_source": "llm_error",
+ "adjudicated_failure_subtype": "thresholds_rates",
+ "adjudicated_on": "2026-11-11",
+ "adjudicator": "developer",
+ "judge_reference_suspect": false,
+ "reasoning": "The judge labeled eight models' rows reference_later_law (deepseek-v4-pro-0813, gemini-3-flash-preview, gemini-3.5-flash, gemini-3.6-flash, gemini-3.7-flash, gpt-5.5, grok-4.5 and grok-build-0.1) because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, a personal exemption and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, at the 2026 single amounts of Rev. Proc. 2025-32, released October 9, 2025 (a $16,100 standard deduction, with the 12% rate starting at $12,400 and the 22% rate at $50,400), and the benchmark scores law published before the freeze; the rows are model errors, as the 2026-09-29 and 2026-10-14 waves decided for the same rule in scenario_007's and scenario_009's federal income tax.",
+ "judged_on_utc": "2026-11-11"
}
]
}
diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv
index 5e75a941..cfb9074c 100644
--- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv
+++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv
@@ -1,50 +1,51 @@
country,scenario_id,variable,model,failure_source,failure_subtype,reference_suspect,annotation
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"Treated the head as a qualifying surviving spouse even though no dependent child is listed. It then used the $32,000/$44,000 joint Social Security thresholds, which cut taxable SS to $11,542 instead of $18,542.70, and applied the ~$33,950 joint standard deduction instead of the $18,150 single amount. That left only $3,486 taxable instead of $26,287.11."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Got AGI right at $50,436. It then applied the $32,200 + $1,650 QSS/joint standard deduction, although a surviving spouse without a dependent child files single and gets $16,100 + $2,050. As a result it taxed only $10,586 instead of $26,287.11, all at 10%."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Put the $48,000 of nontaxable veterans benefits into gross income, taxed 85% of Social Security, and subtracted employer insurance premiums from AGI. It then dropped its own ~$6,600 estimate and claimed $0 with no computation. The correct single-filer result is $26,287.11 taxable and $2,906.45 of tax."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Claimed taxable income falls below zero, but AGI is $50,437.11 ($31,894.41 of pension/IRA/interest plus $18,542.70 of taxable SS). Total deductions are only $24,150 (the $18,150 single aged standard deduction plus the $6,000 senior deduction), which leaves $26,287.11 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Picked single status and the right deductions, but used 50% of benefits ($13,048) as the first tier of taxable SS instead of the $4,500 cap (half of the $9,000 gap between the thresholds). That pushed taxable SS to the $22,182 maximum instead of $18,542.70. It also ended the 10% bracket at $12,150 instead of $12,400, giving $29,926 taxable and $3,348 of tax."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"Used qualifying-surviving-spouse status and joint deductions even though there is no dependent. Even by its own numbers ($54,076 AGI less ~$40,000 of deductions, about $14,000 taxable), tax is positive, yet it asserted taxable income is near zero and answered $0."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Taxed the full 85% of Social Security ($22,182) instead of the worksheet amount of $18,542.70, which overstated AGI at $54,076. It then computed about $3,300 of tax and swapped in an unsupported $1,176. The correct taxable income is $26,287.11, which gives $2,906.45."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"Got AGI right at $50,436 but applied the $32,200 joint/QSS standard deduction plus the $1,650 married aged add-on. A surviving spouse without a dependent child files single, with a $16,100 deduction plus a $2,050 aged add-on. That understated taxable income at $10,586 instead of $26,287.11."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Treated the head as a qualifying surviving spouse, so it used joint thresholds and taxed 85% of SS ($22,182), and applied a ~$31,550 joint standard deduction. It also left out the $6,000 senior deduction. As a single filer, taxable income is $26,287.11."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"Used QSS/joint status, taxed 85% of SS ($22,182), applied a $33,850 joint standard deduction, and left out the $6,000 senior deduction. It then inflated its own $2,023 result to an unsupported $3,204. As a single filer, taxable income is $26,287.11."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"Correctly used single thresholds for Social Security (AGI of $50,436), then mixed in the $32,200 + $1,650 surviving-spouse/joint standard deduction instead of the $18,150 single aged deduction. That cut taxable income to $10,586 instead of $26,287.11."
-us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Got AGI right at $50,436, but used an understated $15,400 single base standard deduction (the 2026 amount is $16,100). It also left out the OBBBA $6,000 senior deduction for filers 65 and older, which made taxable income $32,986 instead of $26,287.11."
-us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Taxed the full 85% of Social Security ($22,182) instead of the worksheet amount of $18,542.70. It also used a $17,350 standard deduction instead of $18,150 and left out the $6,000 senior deduction, which gave $36,726 taxable instead of $26,287.11."
-us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Assumed TCJA had expired and applied a $5,300 personal exemption and a 15% bracket. Under OBBBA, 2026 has no personal exemption and uses 10%/12% brackets. It also counted the employer insurance premiums as an itemized medical expense and left out the $18,150 standard deduction plus the $6,000 senior deduction, which give $26,287.11 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Got AGI ($50,436) and the $18,150 single aged standard deduction right, but left out the OBBBA $6,000 senior deduction, which made taxable income $32,286 instead of $26,287.11. It also ended the 10% bracket at $12,200 instead of the 2026 figure of $12,400."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Assumed the TCJA sunset and applied a $5,300 personal exemption and a 15% second bracket, although OBBBA made the TCJA structure permanent. It also treated the employer insurance premiums as a deductible medical expense, and it never applied the $18,150 standard deduction plus the $6,000 senior deduction."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Stated taxable income of $37,990, which leaves out the $6,000 senior deduction. The $1,109 tax it reported does not even match that figure at 2026 single rates. The correct taxable income is $26,287.11 ($50,437.11 AGI minus $18,150 minus $6,000)."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Applied pre-TCJA rules (itemized real estate taxes, a ~$5,400 personal exemption, and a 15% bracket) even though OBBBA kept the TCJA structure for 2026. The $18,150 single aged standard deduction plus the $6,000 senior deduction exceeds its $21,870 total and gives $26,287.11 taxed at 10%/12%."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Assumed the TCJA had reverted for 2026 and applied a personal exemption, itemized real estate taxes, and a 15% bracket. OBBBA made the standard deduction ($16,100 + $2,050 aged) permanent, added the $6,000 senior deduction, and kept the 12% bracket."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Claimed income falls below the deductions, but AGI including $18,542.70 of taxable SS is $50,437.11, while deductions total only $24,150. That leaves $26,287.11 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Used the expired-TCJA framework of a personal exemption plus itemized real estate taxes. OBBBA has no personal exemption in 2026, and the $18,150 single aged standard deduction plus the $6,000 senior deduction applies, which gives $26,287.11 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Applied pre-TCJA law (a personal exemption, itemized real estate taxes, and a 15% bracket) instead of the OBBBA 2026 rules. Those rules use the $18,150 standard deduction plus the $6,000 senior deduction and 10%/12% brackets."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Got AGI right at $50,436, but used a $17,000 standard deduction instead of $18,150 and left out the OBBBA $6,000 senior deduction. That gave $33,436 taxable instead of $26,287.11."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"Assumed the TCJA had expired and applied a $5,300 personal exemption and a 15% bracket. It also miscomputed taxable SS as $19,942 instead of $18,542.70 and counted employer premiums as medical expenses. It never applied the $18,150 standard deduction plus the $6,000 senior deduction."
-us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Got single status and AGI ($50,436) right, but used an estimated $17,510 standard deduction instead of $18,150 and left out the OBBBA $6,000 senior deduction. That gave $32,926 taxable instead of $26,287.11."
-us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Treated the $8,389 of employer insurance premiums as an itemized medical expense, which inflated itemized deductions to $21,076 and made itemizing look better. It also left out the $6,000 senior deduction. Without the premiums, itemized is $17,587.20, so the $18,150 standard deduction plus the $6,000 senior deduction applies."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"Relied on a surviving-spouse rate structure that does not apply without a dependent child. It also asserted full offset, but single deductions of $24,150 against $50,437.11 of AGI leave $26,287.11 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Asserted the aged standard deduction wipes out all income. In fact $50,437.11 of AGI less $18,150 standard and $6,000 senior deductions leaves $26,287.11 taxable, which gives $2,906.45."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"Applied surviving-spouse/joint thresholds, which cut taxable SS to $11,542, and used a $33,850 joint standard deduction. A surviving spouse without a dependent child files single, which gives $18,542.70 of taxable SS and an $18,150 standard deduction."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Used the correct single-filer structure, but overstated taxable Social Security at about $18,592 instead of $18,542.70. That made taxable income $26,336 instead of $26,287.11 and the tax $2,912.34 instead of $2,906.45."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Used surviving-spouse (joint) status, with joint SS thresholds and the joint standard deduction, which left only about $3,590 taxable. Filing single gives $26,287.11 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"Got AGI right at $50,436 but applied the $33,850 surviving-spouse/joint standard deduction instead of the $18,150 single aged deduction. That cut taxable income to $10,586 instead of $26,287.11."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Used the correct single structure, but carried taxable income of $26,336 instead of $26,287.11 because it overstated taxable Social Security by about $50. That produced $2,912 instead of $2,906.45."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"Applied surviving-spouse joint thresholds, which gave $11,542 of taxable SS, and a $33,850 joint standard deduction. Filing single gives $18,542.70 of taxable SS and an $18,150 standard deduction."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,household_unit_or_filing_status,False,"Used surviving-spouse filing status, which brings in the ~$33,850 joint standard deduction and leaves $10,586 taxable. A filer without a dependent child files single, with an $18,150 standard deduction and $26,287.11 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Asserted senior deductions exceed taxable income, but the $18,150 standard deduction plus the $6,000 senior deduction against $50,437.11 of AGI leaves $26,287.11 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Assumed post-TCJA-sunset law with a $5,300 personal exemption and a 15% bracket, and left out the $6,000 senior deduction. It also applied the $23,200 joint 10% bracket under qualifying-widow status, although there is no dependent child, so the head files single with a 10% bracket ending at $12,400."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Assumed TCJA expired and used a personal exemption, itemized property taxes, and a 15% bracket. In 2026 there is no personal exemption, and the $18,150 standard deduction plus the $6,000 senior deduction gives $26,287.11 taxed at 10%/12%."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Got every step right except the bracket: it ended the 10% bracket at $12,200 instead of the 2026 single threshold of $12,400. Taxing that extra $200 at 12% added about $4 and produced $2,910 instead of $2,906.45."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Assumed the TCJA had sunset, applying a 15% bracket and an uncapped SALT deduction, and counted the $8,389 of employer premiums as a medical deduction. It also left out the $6,000 senior deduction. The correct path is the $18,150 standard deduction plus $6,000, which gives $26,287.11 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no answer or reasoning for federal_income_tax_before_refundable_credits, so there is no value to score."
-us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Subtracted a $750 elderly credit and an inapplicable education credit to reach zero. The elderly credit is zero because nontaxable Social Security ($7,553) exceeds the $5,000 single base amount. No nonrefundable credits offset the $2,906.45 tax."
-us,scenario_000,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Got AGI ($50,436), the $18,150 standard deduction, and the 2026 brackets right, but left out the OBBBA $6,000 senior deduction for filers 65 and older. That left $32,286 taxable instead of $26,287.11."
-us,scenario_000,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Left half of Social Security out of provisional income ($37,472 instead of $50,520) and also dropped the $4,500 first-tier amount, so taxable SS came out at $2,951 instead of $18,542.70. That understated AGI at $34,845."
-us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"Claimed no Social Security is taxable, but provisional income of $50,520 exceeds $34,000, so $18,542.70 is taxable. It also used a $31,150 QSS/joint standard deduction, although without a dependent child the head files single."
-us,scenario_000,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Asserted the single aged standard deduction eliminates taxable income. With $50,437.11 of AGI against $24,150 of standard and senior deductions, $26,287.11 is taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"Used qualifying-surviving-spouse status even though there is no dependent child, which gave it the $32,300 joint standard deduction. It also applied the $32,000/$44,000 joint Social Security thresholds, which QSS does not use, and so taxed only $11,542 of benefits instead of $18,542.70. The correct filing status is Single, with $18,150 plus $6,000 of deductions and $26,287.11 of taxable income."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"Got taxable Social Security ($18,542) and AGI right but filed as qualifying surviving spouse without a dependent child. That let it use the $32,200 joint standard deduction instead of the single $16,100 base (plus the $2,050 aged add-on), cutting taxable income to $10,586 instead of $26,287.11."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,household_unit_or_filing_status,False,"Got AGI of $50,436 right but used surviving-spouse status, and the joint $32,200 + $1,650 standard deduction, for a filer with no qualifying child. The Single deduction is $16,100 + $2,050 + $6,000. The extra ~$15,700 of deduction pushed its taxable income down to $10,586."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Counted nontaxable VA benefits as income, taxed 85% of Social Security and subtracted ESI premiums from AGI. Its own working reached about $6,600 of tax, yet it submitted $0 with no supporting calculation. The correct taxable income is $26,287.11 after the single standard, aged and senior deductions, which produces $2,906.45."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Asserted that the deductions wipe out taxable income without computing it. AGI of $50,437.11, including $18,542.70 of taxable Social Security, minus $24,150 of standard, aged and senior deductions leaves $26,287.11 of taxable income, not zero."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"In the second tier of the Social Security formula it added half of benefits ($13,048) instead of the lesser of $4,500 or half of benefits. That taxed $22,182 instead of $18,542.70 and inflated AGI to $54,076. It also ended the 10% bracket at $12,150 instead of the 2026 single $12,400."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"Treated the head as a qualifying surviving spouse, which requires a dependent child, and applied the joint deduction of about $40,000. It also taxed 85% of benefits, and then declared taxable income 'near zero' even though its own figures left about $14,000 taxable. As a Single filer the head has $26,287.11 of taxable income."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Taxed $22,182 of Social Security by adding half of benefits instead of the $4,500 tier amount, which overstated AGI at $54,076. It then submitted $1,176, which its own ~$29,900 taxable-income calculation does not support. The correct taxable SS is $18,542.70, giving $26,287.11 of taxable income and $2,906.45 of tax."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"Filed as surviving spouse with the $32,200 joint standard deduction even though there is no qualifying dependent. Single status gives $16,100 + $2,050 aged + $6,000 senior = $24,150 of deductions, so taxable income is $26,287.11, not $10,586."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Used QSS status without a dependent child, the joint standard deduction (about $31,550) and the $32,000/$44,000 joint Social Security thresholds, taxing 85% of benefits. It also omitted the $6,000 senior deduction. As a Single filer, taxable SS is $18,542.70 and taxable income is $26,287.11."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"Applied QSS status and its joint $33,850 standard deduction to a filer with no qualifying child, used the joint SS thresholds to tax 85% of benefits, and omitted the $6,000 senior deduction. It then inflated its own computed $2,023 to an unsupported $3,204."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"Computed taxable SS of $18,542 correctly but took the joint-sized $32,200 + $1,650 surviving-spouse standard deduction. Without a dependent child the head files Single, with $16,100 + $2,050, so taxable income is $26,287.11 rather than $10,586."
+us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Left out the OBBBA $6,000 senior deduction for filers aged 65 and over and understated the 2026 single base standard deduction ($15,400 instead of $16,100). As a result it taxed $32,986 instead of $26,287.11."
+us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Taxed 85% of Social Security ($22,182) instead of the $18,542.70 the two-tier formula yields. It also used a $17,350 standard deduction instead of $18,150 and omitted the $6,000 senior deduction, which pushed taxable income to $36,726."
+us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Assumed the TCJA had expired in 2026, so it used a personal exemption and a 15% bracket, although OBBBA made the TCJA rates and standard deduction permanent. It also treated the ESI premiums as an itemizable medical expense and omitted the $6,000 senior deduction. The correct deductions are the $18,150 standard deduction plus $6,000, with 10%/12% brackets."
+us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Correctly used the $18,150 single aged standard deduction but omitted the $6,000 OBBBA senior deduction, leaving $32,286 taxable instead of $26,287.11. It also ended the 10% bracket at $12,200 instead of $12,400."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Assumed the TCJA had expired in 2026 and applied a $5,300 personal exemption with a 15% bracket, but OBBBA made the TCJA structure permanent. It also counted the ESI premiums as a medical deduction and omitted the standard and senior deductions, which total $24,150."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Reported $37,990 of taxable income, which omits the $6,000 senior deduction and the full $18,150 single aged standard deduction, and then gave a tax of $1,109 that does not follow from that base. The correct taxable income is $26,287.11."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied pre-TCJA 2026 law, with about $5,400 of personal exemption, itemized real estate taxes and a 15% bracket. Under OBBBA, the $18,150 standard deduction plus the $6,000 senior deduction beats itemizing, and the 10%/12% brackets apply."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Assumed the TCJA had reverted in 2026: it itemized $16,470, took a $5,050 personal exemption and used a 15% bracket. OBBBA made the higher standard deduction ($16,100 + $2,050 aged), the $6,000 senior deduction and the 12% bracket permanent."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Claimed income was below deductions without computing taxable Social Security. AGI is $50,437.11 against $24,150 of single standard, aged and senior deductions, leaving $26,287.11 of taxable income."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Applied post-TCJA-sunset rules: it itemized property taxes, added a personal exemption and taxed above the 10% bracket at 15%. OBBBA retained the larger standard deduction, the 10%/12% brackets and the $6,000 senior deduction, which yield $26,287.11 of taxable income."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Used reverted pre-TCJA law, with itemized real estate taxes, a personal exemption and a 15% bracket. OBBBA kept the $16,100 + $2,050 standard deduction, added the $6,000 senior deduction and kept the 12% second bracket."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Used a $17,000 single aged standard deduction instead of $18,150 and omitted the $6,000 OBBBA senior deduction, leaving $33,436 taxable instead of $26,287.11."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"Assumed TCJA expiration (personal exemption and a 15% bracket) and counted ESI premiums as a medical deduction. It also misstated taxable Social Security as $19,942; the two-tier formula gives $18,542.70. The correct return uses the $18,150 standard deduction plus the $6,000 senior deduction and 10%/12% brackets."
+us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Computed taxable Social Security and AGI correctly as a Single filer but omitted the OBBBA $6,000 senior deduction and estimated the standard deduction at $17,510 instead of $18,150. That left $32,926 taxable instead of $26,287.11."
+us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Counted the employer-sponsored insurance premiums as an itemizable medical expense, giving itemized deductions of $21,076 instead of the $18,150 standard deduction. It also omitted the $6,000 senior deduction, which applies whether or not the filer itemizes, so it taxed $29,360 instead of $26,287.11."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"Relied on a surviving-spouse rate structure that requires a dependent child, then asserted zero tax without computing it. As a Single filer the head has $26,287.11 of taxable income after the $24,150 of deductions."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Asserted that the standard deduction fully offsets income. The $24,150 of standard, aged and senior deductions leaves $26,287.11 of the $50,437.11 AGI taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"Used surviving-spouse status and the joint $32,000/$44,000 Social Security thresholds, taxing only $11,542 of benefits, then applied the joint $32,200 standard deduction. The head has no dependent, so the Single thresholds apply (giving $18,542.70 taxable SS) and the deduction is $16,100 + $2,050 + $6,000."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Used the correct Single framework and deductions but overstated taxable Social Security at about $18,592 instead of $18,542.70. That raised taxable income to $26,336 instead of $26,287.11, and tax by about $6."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Used surviving-spouse status, with joint-sized standard deductions and Social Security thresholds, even though there is no qualifying child. That left only about $3,590 in the 10% bracket instead of the $26,287.11 of taxable income a Single filer has."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"Got AGI of $50,436 right but applied the $33,850 surviving-spouse standard deduction. With no dependent child, the head takes the Single $18,150 standard deduction plus the $6,000 senior deduction, which leaves $26,287.11 taxable rather than $10,586."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Used the correct Single deductions but reached $26,336 of taxable income instead of $26,287.11 by overstating taxable Social Security by about $49. It also rounded the result to a whole dollar, giving $2,912 instead of $2,906.45."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"Applied surviving-spouse status with the joint $32,000/$44,000 Social Security thresholds, taxing $11,542 instead of $18,542.70, and the $33,850 joint standard deduction. The head has no qualifying child and files Single."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,household_unit_or_filing_status,False,"Its $1,058.60 equals 10% of $10,586, which shows it used the joint-sized surviving-spouse standard deduction ($32,200 + $1,650) plus $6,000. A filer with no dependent child files Single, with $16,100 + $2,050 + $6,000, and has $26,287.11 of taxable income."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Asserted that senior deductions and credits exceed taxable income without computing them. The $24,150 of standard, aged and senior deductions leaves $26,287.11 of the $50,437.11 AGI taxable, and no elderly credit is available."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Assumed post-TCJA-sunset law (a $5,300 personal exemption, a lower standard deduction and a 15% bracket) and treated the head as a qualifying surviving spouse. It also omitted the $6,000 senior deduction. The Single return under OBBBA has $26,287.11 taxed at 10%/12%."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Applied a restored personal exemption and the pre-TCJA 15% bracket for 2026 and itemized property taxes. OBBBA kept the $18,150 single aged standard deduction and the 12% bracket and added the $6,000 senior deduction."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Got the filing status and every deduction right but ended the 2026 single 10% bracket at $12,200 instead of $12,400, which overstated tax by $4. It then rounded the result to a whole dollar."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Assumed the TCJA had sunset, so it used a 15% bracket and uncapped itemizing, and counted the ESI premiums as a $4,606 medical deduction. It also omitted the $6,000 senior deduction. Under OBBBA the head takes the $18,150 standard deduction plus $6,000, with 10%/12% brackets."
+us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for this variable, so there was no answer to score."
+us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Subtracted a $750 elderly credit and an irrelevant lifetime learning credit to reach zero. The elderly credit is fully phased out by the head's nontaxable Social Security and AGI, so tax on $26,287.11 of taxable income is $2,906.45."
+us,scenario_000,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Got taxable Social Security, AGI, the $18,150 single aged standard deduction and the brackets right, but omitted the OBBBA $6,000 senior deduction for filers 65 and over. That left $32,286 taxable instead of $26,287.11."
+us,scenario_000,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Left half of benefits out of provisional income (using $37,472 instead of $50,520) and dropped the $4,500 first-tier amount, so it taxed only $2,951.20 of Social Security instead of $18,542.70. Its deductions were correct, but taxable income was understated at $10,695.20."
+us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Treated Social Security as untaxed even though provisional income of $50,520 exceeds the $34,000 threshold, which makes $18,542.70 taxable. It also used a joint-sized QSS standard deduction for a filer with no dependent and omitted the aged and senior deductions."
+us,scenario_000,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Claimed the single aged standard deduction eliminates taxable income. AGI of $50,437.11, including $18,542.70 of taxable Social Security, minus the $18,150 standard deduction and the $6,000 senior deduction leaves $26,287.11 taxable."
us,scenario_000,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_000,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated age 77 and an asserted income exclusion calculation as sufficient for Texas aged Medicaid without identifying or satisfying an aged eligibility pathway. The person qualifies through no Medicaid category and receives no SSI, so the claimed below-threshold aged eligibility does not apply."
us,scenario_000,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no earned wages were reported and that payroll tax therefore equals zero, but then submitted $1,534 instead. Its numeric output contradicts its own payroll-tax-base analysis and the required final-value contract."
@@ -89,45 +90,46 @@ us,scenario_002,federal_income_tax_before_refundable_credits,muse-wonder,prompt_
us,scenario_002,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"The model excluded the disability benefits in its final pass but declared the full $44,080 of Social Security taxable. Its own combined income of $40,780 falls below the $44,000 MFJ second threshold, so only 50% × ($40,780 − $32,000) = $4,390 is taxable. It also used a $32,600 deduction that leaves out the $12,000 senior deduction. The correct AGI of $23,130 gives $0 tax."
us,scenario_002,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified a zero taxable wage base and then contradicted that computation by inventing taxable wage-equivalent income and assigning $1,926 of Social Security and Medicare taxes. No listed income is employee compensation subject to payroll tax, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll-tax components all equal zero."
us,scenario_002,spouse_medicaid_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly evaluated the spouse as a separate zero-income individual and assumed that age alone opened an aged Medicaid pathway. Washington's pathway evaluation includes the applicable household income and categorical requirements; this spouse receives no SSI, has household MAGI of 2.90 times FPL, and qualifies for no category."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own reasoning reached about $22,379, close to the correct figure but using a $32,300 standard deduction instead of $32,200. It then submitted $27,049, which doesn't follow from that work. The submitted number matches a TCJA-sunset calculation (personal exemptions and 25% bracket), and 2026 law does not use those rules."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It subtracted $500 of auto-loan interest as a deduction on top of the $32,200 standard deduction. That $500 is ordinary household auto-loan interest, not qualified vehicle-loan interest, so it isn't deductible here. The correct taxable income is $151,090.25, not $150,590.50, and the extra deduction cut the tax by exactly 22% × $500 = $110."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It deducted the traditional 401(k) ($23,926) and IRA ($1,118) contributions even though there are no wages or other compensation to support them. It also figured taxable Social Security as 85% of the excess over $32,000 ($23,384) instead of the $50,583 cap. Finally, it used a $28,900 standard deduction with an over-65 addition, but neither spouse is 65 and the 2026 MFJ amount is $32,200."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave a rough $14,800 without working through the brackets and was unsure about 65+ status. Neither spouse is 65, so no additional or senior deduction applies. On $151,090.25 of taxable income, the 2026 MFJ brackets plus 15% on qualified dividends produce $22,400.65."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its reasoning reached about $22,652 using 2025-level brackets (10% to $24,150, 12% to $98,200). It then submitted an unsupported $19,524, citing 'estimation uncertainty'. The correct 2026 brackets (10% to $24,800, 12% to $100,800) on $147,330.25 of ordinary income give $21,836.65, plus $564 on dividends."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used 2025 bracket thresholds (10% to $23,850, 12% to $96,950) and a $32,300 standard deduction. The 2026 MFJ values are $24,800, $100,800, and $32,200. Because the lower thresholds push more income into the 22% bracket, it overstated tax by about $380."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It said it applied a 24% bracket to about $147,331 of ordinary income and got $23,886. The 24% bracket doesn't start until $211,400 for 2026 MFJ, so all of the income above $100,800 belongs at 22%. Correct ordinary tax is $21,836.65."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It projected the 2026 standard deduction as $30,000 by inflating the 2024 amount. That misses the OBBBA-set $32,200 and overstates taxable income by $2,200. It also used estimated brackets (12% ending at $99,375) instead of the actual $100,800 threshold."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It deducted the $1,118 traditional IRA contribution. A deductible IRA contribution requires taxable compensation, and this household has no wages. It also overstated pre-Social Security income by $100 ($132,807 instead of $132,707), and then rounded a rough bracket estimate down to $22,000."
-us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It cut AGI by $23,500 of 401(k) deferrals and $1,118 of IRA contributions. With no wages there is nothing to defer and no compensation to support an IRA deduction, so AGI stays at $183,290.25. It also used a $30,600 standard deduction instead of $32,200."
-us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA law to 2026: $5,100 personal exemptions, a $15,000 standard deduction, and 10/15/25% brackets. OBBBA made the TCJA structure permanent, so 2026 uses the $32,200 standard deduction, no exemptions, and 10/12/22% brackets. It also dropped about $2,680 from AGI ($180,611 instead of $183,290)."
-us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It claimed Social Security is not taxed in 2026 and left out the full $50,583 taxable portion. Benefits remain taxable under the provisional-income test: provisional income of about $167,704 is far above $44,000, so 85% is includible. It also used a $30,700 standard deduction instead of $32,200."
-us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It got AGI right but used a $30,000 standard deduction instead of $32,200. It also used 2025 bracket thresholds (10% to $23,850, 12% to $96,950) instead of the 2026 values ($24,800 and $100,800). Both errors overstated tax."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expires in 2026 and used a $16,000 standard deduction, $10,000 of personal exemptions, and 10/15/25% brackets. OBBBA made the TCJA permanent, so the correct calculation uses the $32,200 standard deduction, no exemptions, and 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It put taxable income at about $183,865, which is roughly AGI. In effect, it never subtracted the $32,200 standard deduction. Taxable income is $151,090.25, and that produces $22,400.65 of tax."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expires and subtracted a pre-TCJA standard deduction plus personal exemptions, then applied the older bracket structure. Under OBBBA-permanent law, 2026 has a $32,200 standard deduction, no personal exemptions, and 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: a $16,100 standard deduction, $10,100 of personal exemptions, and the 25% bracket. OBBBA made the TCJA permanent, so the correct 2026 calculation uses a $32,200 standard deduction, no exemptions, and 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It reported $0 without any calculation, even though AGI of $183,290.25 is far above the $32,200 standard deduction and there are no offsetting nonrefundable credits. The $151,090.25 of taxable income produces $22,400.65 of tax."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of $32,200. It also used 2025 bracket thresholds (10% to $23,850, 12% to $96,950) instead of the 2026 values ($24,800 and $100,800). The result was about $712 too high."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It subtracted personal exemptions and a pre-TCJA standard deduction, which assumes the TCJA sunset. 2026 law has no personal exemptions, a $32,200 MFJ standard deduction, and 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It got AGI right, but its $23,287 corresponds to a standard deduction of about $30,000 and 2025 bracket thresholds. The correct 2026 parameters are a $32,200 standard deduction and brackets of $24,800 and $100,800, which give $22,400.65."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"Its $22,290.71 is exactly 22% × $500 below the correct amount. That shows it subtracted the $500 of household auto-loan interest on top of the standard deduction. This interest is not deductible here, so taxable income stays at $151,090.25."
-us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It made several errors: it applied a -$3,000 long-term loss instead of the actual $2,662, overstated ordinary tax at about $28,803 on roughly $152k, and invented a $500 credit for a sick spouse. It then submitted $49,523.45, which contradicts its own $28,867 figure."
-us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It added the $1,080 short-term gain to ordinary income and then also subtracted the $1,582 net capital loss, which already includes that gain. That double count overstated AGI at $184,370.50 instead of $183,290.25. It also used a $32,600 standard deduction instead of $32,200."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"It assumed age-65 deductions and itemized medical and interest deductions wiped out the tax. The head is 64 and the spouse is 61, so neither the additional standard deduction nor the senior deduction applies. Medical expenses of $10,900 fall below the 7.5%-of-AGI floor, so the $32,200 standard deduction leaves $151,090.25 taxable."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It reduced income using retirement-account deductions and itemized medical expenses. With no compensation, the contributions aren't deductible, and the $10,900 of medical expenses is below the $13,747 floor (7.5% of AGI). With only the $32,200 standard deduction, taxable income is $151,090.25 and the tax is $22,400.65, not $2,320."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It treated the desired retirement contributions and the standard deduction as enough to zero out the tax. With no wages, none of the contributions are deductible. The $32,200 standard deduction leaves $151,090.25 taxable."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used post-TCJA-sunset rules: a $16,100 standard deduction, about $10,600 of personal exemptions, and 10/15/25% brackets. OBBBA made the TCJA permanent, so 2026 uses a $32,200 standard deduction, no exemptions, and 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It got AGI right but applied a pre-TCJA standard deduction, personal exemptions, and 10/15/25% brackets on the assumption that the TCJA sunset. 2026 law keeps the $32,200 standard deduction, no exemptions, and 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and used a $17,000 standard deduction plus $10,800 of personal exemptions. OBBBA made the TCJA permanent, so the 2026 MFJ standard deduction is $32,200 with no exemptions, and the 22% bracket applies above $100,800."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used inflation-adjusted pre-TCJA parameters: a $17,103 standard deduction, $10,908 of exemptions, and 10/15/25% brackets. 2026 law under OBBBA keeps the $32,200 standard deduction, no exemptions, and 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It counted the $1,080 short-term gain as income and then also subtracted the full $1,582 net capital loss, which already includes that gain. That double count overstated AGI at about $184,371 instead of $183,290.25, and taxable income by $1,080."
-us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used a projected pre-TCJA $16,950 standard deduction, $10,800 of personal exemptions, and 10/15/25% brackets. OBBBA made the TCJA permanent, so 2026 uses the $32,200 standard deduction, no exemptions, and 10/12/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted a $500 car-loan-interest deduction, but the household's auto-loan interest is not deductible here. It also used understated bracket thresholds (12% ending around $99,150 instead of $100,800). Ordinary tax on $147,330.25 is $21,836.65."
-us,scenario_003,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It returned $0 because there were no wages. Retirement distributions, pensions, taxable Social Security, interest, and dividends add up to $183,290.25 of AGI. After the $32,200 standard deduction, the tax is $22,400.65."
-us,scenario_003,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used the correct $32,200 standard deduction, but its $22,241 ordinary tax comes from 2025 bracket thresholds (10% to $23,850, 12% to $96,950). The 2026 thresholds ($24,800 and $100,800) give $21,836.65."
-us,scenario_003,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted the $1,582 net capital loss a second time after it was already netted into AGI. That put taxable income at $149,508.50 instead of $183,290.50 - $32,200 = $151,090.50, which understated the 22%-bracket income."
-us,scenario_003,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It treated net capital gains as $0 instead of taking the $1,582 net capital loss deduction, which overstated AGI at $184,872.50. It used a $32,300 standard deduction based on a wrong '65+ head' premise and overstated ordinary tax at $24,967.98. It then submitted $41,228.50, which contradicts its own $25,531.98 total."
-us,scenario_003,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"Its $21,902.90 is $497.75 below the correct $22,400.65. That gap equals about $2,262 of 22%-bracket income removed by an extra deduction the facts don't support. The correct calculation taxes $147,330.25 of ordinary income through the 2026 MFJ brackets plus $3,760 of qualified dividends at 15%."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own reasoning reached about $22,379, only slightly off because it used a $32,300 standard deduction instead of $32,200. It then submitted $27,049, a number that does not follow from its own calculation."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It subtracted a $500 auto-loan-interest deduction. The household does not qualify for OBBBA's vehicle-loan-interest deduction because no qualifying new, US-assembled vehicle loan is listed. That understated taxable income by $500 and tax by about $110."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It subtracted $500 of auto-loan interest along with the $32,200 standard deduction, but no qualifying vehicle loan supports that OBBBA deduction. Taxable income came out at $150,590.50 instead of $151,090.25, so tax was about $110 too low."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It counted only $23,384 of Social Security as taxable (85% of the amount over $32,000) instead of $50,583 (85% of benefits). It also deducted the 401(k) and IRA contributions even though there is no earned income, and used an outdated $28,900 standard deduction that included an age-65 add-on neither spouse qualifies for."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It gave no real calculation and was unsure whether the couple qualified for age-65 treatment. Neither spouse is 65. The correct path gives $151,090 of taxable income and $22,401 of tax; its $14,800 implies about $36,000 less taxable income, which matches deducting the 401(k)/IRA contributions or understating taxable Social Security."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It calculated about $22,652–$22,674, using 2025-style bracket thresholds and a $32,300 standard deduction. It then submitted $19,524 'given estimation uncertainty', a figure its own calculation does not support."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used 2025 bracket thresholds (10% to $23,850, 12% to $96,950) instead of 2026 (10% to $24,800, 12% to $100,800), and a $32,300 standard deduction instead of $32,200. That raised ordinary-rate tax to $22,219 versus $21,837."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Its taxable income of about $151,091 was right, but it applied a 24% bracket. Joint filers do not reach 24% until above roughly $211,000, so $147,331 of ordinary income is taxed only at 10/12/22%. It reported $23,886 of ordinary tax instead of $21,837."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It estimated the 2026 joint standard deduction at $30,000 by inflating pre-OBBBA figures, missing OBBBA's $32,200. It also used projected bracket thresholds ($24,450 and $99,375) instead of $24,800 and $100,800, which overstated taxable income by $2,200 and tax by about $633."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It deducted the $1,118 traditional IRA contribution, but an IRA deduction requires earned compensation and there are no wages. It also misadded non-Social Security income as $132,807 instead of $132,707, then approximated the brackets and rounded down to $22,000."
+us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It cut AGI by $23,500 of 401(k) deferrals and $1,118 of IRA contributions, but with no wages or earned income neither is allowed. It also used a $30,600 standard deduction instead of $32,200."
+us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-TCJA rules (personal exemptions, a $15,000 standard deduction and 10/15/25% brackets), but OBBBA made the TCJA structure permanent. For 2026 the correct rules are the $32,200 standard deduction, no exemptions, and 10/12/22% brackets. It also miscounted AGI as $180,611."
+us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It wrongly claimed Social Security is excluded from income in 2026 and left out the $50,583 of taxable benefits. Provisional income far exceeds $44,000, so 85% of benefits is taxable. It also used a $30,700 standard deduction instead of $32,200."
+us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction instead of OBBBA's $32,200, and 2025 bracket thresholds ($23,850 and $96,950) instead of 2026's $24,800 and $100,800. Both errors overstated tax."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA had expired and applied a $16,000 standard deduction, $10,000 of personal exemptions and 10/15/25% brackets. Under OBBBA the 2026 rules are a $32,200 standard deduction, no exemptions, and 10/12/22% brackets."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used about $183,865 as taxable income, which is roughly AGI and even exceeds the true AGI of $183,290. So it never actually subtracted the $32,200 standard deduction, and tax came out at $31,317 instead of $22,401."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly assumed the TCJA had expired and subtracted a pre-TCJA standard deduction plus personal exemptions, then taxed the result at pre-TCJA rates. OBBBA kept the $32,200 standard deduction and 10/12/22% brackets for 2026."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied a 2026 TCJA sunset: a $16,100 standard deduction plus $10,100 of personal exemptions and the 15%/25% brackets. Under OBBBA the correct figures are a $32,200 standard deduction with no exemptions and the 12%/22% brackets."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It reported $0 without a calculation. The household has about $183,290 of AGI from IRA distributions, pensions and taxable Social Security, and after the $32,200 standard deduction it owes $22,401."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of OBBBA's $32,200. It also taxed ordinary income at 2025 bracket thresholds ($23,850 and $96,950) instead of 2026's $24,800 and $100,800, producing $23,112.71."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It subtracted personal exemptions and used pre-TCJA rates for 2026, assuming the TCJA had sunset. OBBBA made the TCJA structure permanent, so the correct rules are the $32,200 standard deduction, zero exemptions, and 10/12/22% brackets."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $23,287 matches a roughly $30,000 standard deduction combined with 2025 bracket thresholds ($23,850 and $96,950). The 2026 figures are a $32,200 standard deduction and thresholds of $24,800 and $100,800. It also misadded AGI as $183,280.50."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"Its $22,290.71 is exactly 22% × $500 below the correct tax. That means it subtracted the $500 of auto-loan interest even though the household has no qualifying OBBBA vehicle-loan-interest deduction."
+us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It allowed a $3,000 capital loss when the net loss is only $1,582, and used a $30,000 standard deduction. It applied ordinary rates to the whole taxable income, including the dividends, and made up a $500 nonrefundable credit. It then submitted $49,523.45, which does not match its own $28,867."
+us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It added the $1,080 short-term gain as income and also subtracted the full $1,582 net loss that already included it. That made the net capital effect −$502 instead of −$1,582 and overstated AGI by $1,080. It also used a $32,600 standard deduction instead of $32,200."
+us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,age_disability,False,"It assumed age-65 deductions apply, but the head is 64 and the spouse 61. It also assumed itemized medical and interest deductions wipe out the income, yet the $10,900 of medical costs is below the 7.5%-of-AGI floor of $13,747. The $32,200 standard deduction applies and leaves $22,401 of tax."
+us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It deducted retirement-account contributions even though there is no earned income, and it relied on itemized medical expenses that fall below the 7.5%-of-AGI floor. Its $2,320 implies taxable income far below the correct $151,090."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It offset the retirement income with the desired 401(k)/IRA contributions, which are not deductible without wages, and reported zero tax. After the $32,200 standard deduction, the $183,290 AGI leaves $151,090 taxable and $22,401 of tax."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used post-TCJA-sunset rules: a $16,100 standard deduction, $10,600 of personal exemptions and 10/15/25% brackets. Under OBBBA, 2026 keeps the $32,200 standard deduction with no exemptions and the 10/12/22% brackets."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its AGI of $183,291 was right, but it then used a post-TCJA-sunset standard deduction, personal exemptions and 10/15/25% rates. OBBBA made the $32,200 standard deduction and 12%/22% brackets permanent for 2026."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It assumed the TCJA had sunset, using a $17,000 standard deduction and $10,800 of personal exemptions. Under OBBBA the correct figures are a $32,200 standard deduction, no exemptions, and 10/12/22% brackets."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used inflation-adjusted pre-TCJA rules: a $17,103 standard deduction, $10,908 of exemptions and 10/15/25% brackets. OBBBA kept the TCJA's $32,200 standard deduction and 10/12/22% brackets for 2026."
+us,scenario_003,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It counted the $1,080 short-term gain as separate income and also subtracted the full $1,582 net capital loss that already nets that gain. That overstated AGI as $184,371 instead of $183,290, so taxable income was $152,171 instead of $151,090."
+us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It applied 'current-law' pre-TCJA projections: a $16,950 standard deduction, $10,800 of personal exemptions and 10/15/25% brackets. OBBBA made the $32,200 standard deduction and 10/12/22% brackets the 2026 law."
+us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It took a $500 car-loan-interest deduction even though the household has no qualifying vehicle loan. It also used bracket thresholds of $24,400 and $99,150 instead of 2026's $24,800 and $100,800."
+us,scenario_003,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It returned $0 because there were no wages, ignoring $94,720 of IRA distributions, a $32,525 pension, investment income and $50,583 of taxable Social Security. Together these produce $151,090 of taxable income and $22,401 of tax."
+us,scenario_003,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"Its taxable income of $151,091 was right, but it taxed ordinary income using 2025 bracket thresholds ($23,850 and $96,950) instead of 2026's $24,800 and $100,800. That gave $22,241 of ordinary-rate tax instead of $21,837."
+us,scenario_003,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted the $1,582 net capital loss twice. Its stated AGI of $183,290.50 already includes the loss, yet it reported taxable income of $149,508.50 instead of $151,090.50. That understated tax by about $348."
+us,scenario_003,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It set net capital gains to $0 instead of deducting the $1,582 loss, used a $32,300 standard deduction, and computed ordinary-rate tax of $24,968 on $148,268, which is about $2,600 too high for 2026 brackets. It then submitted $41,228.50, which does not match its own $25,531.98."
+us,scenario_003,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no breakdown. Its $21,902.90 is $498 below the correct tax, which matches understating taxable income by about $2,260 (an extra deduction beyond the $32,200 standard deduction) rather than $151,090 of taxable income taxed at 10/12/22% plus 15% on the dividends."
us,scenario_003,head_medicare_eligible,claude-haiku-5.5,llm_error,other,False,"The model's explanation correctly said the head, at 64, is under the age-65 Medicare threshold and has no disability pathway, and it ended with ""value = 0"". Its submitted numeric value was still 1 (Yes), so the output field contradicts its own correct reasoning."
us,scenario_003,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model invented a ""near-term eligibility"" rule that counts a 64-year-old as Medicare-eligible. Eligibility requires reaching age 65 (or a disability/ESRD pathway, none of which is present). The prompt also fixes age as constant for the tax year, so no projection to 65 applies."
us,scenario_003,head_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The model's reasoning correctly said that 64 is below the Medicare eligibility age of 65 and that the head is ""not eligible"". It then submitted value = 1, which contradicts its own conclusion; the correct value is 0."
@@ -154,156 +156,159 @@ us,scenario_004,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite
us,scenario_004,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $3,212 implies about $64,000 of New York taxable income under the MFJ brackets — more than the household's entire $52,540 of gross receipts including Social Security — so it both taxed the $51,475 benefit and imputed unlisted wage earnings from the 40-hour work weeks that the prompt requires be treated as 0. The actual New York base is $1,065 of interest less partnership loss, fully offset by the ~$16,050 MFJ standard deduction, for $0 of tax."
us,scenario_004,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It wrote that New York does not tax Social Security yet built its base from $52,540 with the full $51,475 benefit included, never applying the federal provisional-income test ($26,803 against the $32,000 MFJ base) that makes the benefit 0% taxable and therefore leaves only $1,065 of federal AGI to flow to New York. Its arithmetic is also self-contradictory: $52,540 less its stated $16,850 standard deduction is $35,690, not the $18,426 it claimed, and $3,451.12 on $18,426 is an 18.7% effective rate against NY's 4%–4.5% brackets at that level (about $743). The correct result is $1,065 of NY AGI wiped out by the ~$16,050 standard deduction, giving $0."
us,scenario_004,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model correctly found federal AGI under $5,000 and used the table amount of $90 plus $15 for the second exemption, but it counted the NY household credit (Tax Law §606(b)) as refundable. That credit is nonrefundable: it only offsets NY tax, which is zero here, and it never enters state_refundable_credits. None of NY's refundable credits apply (EITC, Empire State child credit, real property tax credit), so the answer is $0, not $105."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Left out the $5,722 NIIT included in the frozen reference; the record leaves its inclusion in this output unresolved. Its own regular-tax build reached $100,206 before an unexplained bump to $100,819. Its $46,308 traditional 401(k) deduction differs from the frozen pro rata $41,650 allocation, whose treatment remains unresolved. It skipped the $1,070 non-itemizer cash charitable deduction and ended the 22% bracket at $211,100 instead of $211,400."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"Its $504,054 taxable income is $559 above the frozen reference's $503,494.75, accounting for a $179 tax difference at 32%. It omitted the $1,070 non-itemizer cash charitable deduction. Its refund inclusion and $46,308 traditional 401(k) deduction differ from the frozen refund exclusion and pro rata $41,650 allocation; those treatments remain unresolved. Both the model and the frozen reference deduct IRA contributions that the recorded correction disallows under 219(g)."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Made up about $110,000 of mortgage interest from the $2.85M balance even though no interest is listed. It then reported $249,914, which no bracket calculation on its own ~$408k taxable income produces. The frozen reference uses the $32,200 standard deduction plus $1,070 charitable, $503,495 of taxable income, and $89,096 ordinary tax + $11,688 qualified-dividend tax + $5,722 NIIT. These figures include a defective $2,164 IRA deduction and remain excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Explicitly skipped the calculation and entered 0 as a placeholder, while the frozen reference reports $503,495 of taxable income and $89,095.92 of ordinary-bracket tax + $11,688 of qualified-dividend tax + $5,721.98 of NIIT. These figures include a defective IRA deduction and remain excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Its own working totals $102,907 of regular tax, but it submitted $91,458, which is about its ordinary-bracket tax alone, so it dropped the $11,688 qualified-dividend tax. It also deliberately left out the $5,722 NIIT included in the frozen reference ('NIIT not part of this line'); inclusion in this output remains unresolved. It used outdated bracket thresholds (24% to ~$396,000) and a $31,500 standard deduction. Its inclusion of the $4,148 refund differs from the frozen calculation, but refund taxation remains unresolved."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"Imputed tens of thousands in mortgage interest from the $2.85M balance (no interest is listed) and applied a $10,000 SALT cap to build ~$60,000 of itemized deductions instead of the $32,200 standard + $1,070 charitable. It taxed qualified dividends at 20% even though taxable income is below the $613,700 threshold where 20% starts. Its stated pieces (~$92,000 + $15,584) don't add up to $154,430."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"Added its own AGI components wrong: they sum to $538,418, not the $604,318 it reported. It then imputed mortgage interest that isn't listed and taxed qualified dividends at an invented 18.8% blended rate. The frozen reference uses $536,764 AGI minus $33,270 of deductions, giving $503,495 of taxable income and $106,506 including NIIT. Its AGI includes a defective $2,164 IRA deduction, and the output remains excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its $504,054 taxable income is $559 above the frozen reference's $503,494.75, accounting for a $179 tax difference at 32%. It omitted the $1,070 non-itemizer cash charitable deduction. Its refund inclusion and $46,308 traditional 401(k) deduction differ from the frozen refund exclusion and pro rata $41,650 allocation; those treatments remain unresolved. Both the model and the frozen reference deduct IRA contributions that the recorded correction disallows under 219(g)."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Assumed the TCJA expires in 2026. It used inflated 2017 brackets (15/25/28/33%), restored 2%-floor miscellaneous deductions for the $13,848 of employee expenses, applied a $10,000 SALT cap, and computed $7,141 of AMT. Under OBBBA the TCJA brackets and the $32,200 standard deduction are permanent, and no AMT is due. It also invented $52,500 of mortgage interest from an assumed 7% rate, and computed NIIT on $153,578 without the $3,000 capital-loss offset."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Itemized $40,574, including about $28,000 of mortgage interest it made up and a $10,000 SALT cap, instead of using the $32,200 standard + $1,070 charitable deduction. Its $115,000 isn't backed by any bracket calculation: its own $497,844 of taxable income produces about $99,000 before NIIT."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Chose to itemize (≈$31.9k phased-down SALT + $2.6k charity), missing that OBBBA's 0.5%-of-AGI floor wipes out the $2,574 of itemized gifts. That leaves itemizing below PolicyEngine's $33,270 ($32,200 standard deduction + $1,070 non-itemizer cash charitable deduction). Its refund inclusion and $46,308 traditional 401(k) deduction differ from the frozen refund exclusion and pro rata $41,650 allocation; those treatments remain unresolved. Omitting the IRA deduction agrees with the recorded correction under 219(g). Its taxable income is about $450 above the excluded frozen figure."
-us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"Used a $30,750 standard deduction instead of 2026's $32,200 and omitted the $1,070 non-itemizer charitable deduction. Its $507,668 taxable income is above the excluded frozen reference's $503,494.75, which includes a defective $2,164 IRA deduction. Refund taxation and the allocation of excess 401(k) deferrals remain unresolved. Its brackets produced $92,234 where the 2026 schedule gives about $90,431 on the same ordinary income, and it computed NIIT on $153,578 without the $3,000 capital-loss offset ($5,836 instead of $5,722)."
-us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"Imposed an AMT of $131,247. Under OBBBA's 2026 AMT parameters ($140,200 MFJ exemption, phase-out starting at $1,000,000), tentative minimum tax is about $96,000, well below regular tax, so no AMT applies. Its $465,449 of taxable income also reflects ~$73k of itemized deductions instead of the $33,270 standard + charitable deduction, and its $5,836 NIIT ignores the $3,000 capital-loss offset."
-us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: itemized deductions with a Pease reduction ($59,513), then an AMT that raised regular tax to $127,938. Under 2026 OBBBA law Pease is repealed and AMT doesn't bind at this income, and PolicyEngine uses the $32,200 standard + $1,070 charitable deduction. Its inclusion of the $4,148 refund differs from the frozen calculation; refund taxation remains unresolved. It computed NIIT on $153,578 without the $3,000 capital-loss offset."
-us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"Taxed $396,990 of ordinary income on pre-TCJA 2026 brackets after $63,508 of itemized deductions. PolicyEngine applies OBBBA's permanent TCJA brackets (24% up to $403,550, then 32%) and the $32,200 standard + $1,070 charitable deduction. The frozen reference's $503,495 taxable income includes a defective $2,164 IRA deduction and remains excluded from scoring. It also put NIIT at $5,836 by ignoring the $3,000 capital-loss offset."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA expiration: it itemized $108,283 with uncapped SALT and mortgage interest on a $1M balance (no interest is listed), and took $10,400 of personal exemptions, producing $88,016 of ordinary tax. Under OBBBA there are no personal exemptions, and the household uses the $32,200 standard + $1,070 charitable deduction. The frozen reference's $503,495 taxable income and $89,095.92 ordinary tax include a defective $2,164 IRA deduction. Its $11,688 QD tax and $5,722 NIIT match the frozen components; the output remains excluded."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"Built a $460,936 AGI that includes the $70,000 tax-exempt pension, which is excluded from gross income, and gave no traceable bracket calculation. The frozen reference's AGI is $536,764 and its taxable income is $503,495 after $33,270 of deductions, producing $89,096 + $11,688 QD tax + $5,722 NIIT. These figures include a defective $2,164 IRA deduction and remain excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employer-sponsored insurance premiums from wages (AGI $525,049). PolicyEngine's wage income is $430,000 minus $41,650 of 401(k) deferrals = $388,350, with no premium exclusion. It then added an AMT that doesn't apply under 2026 OBBBA parameters ($140,200 exemption, $1,000,000 phase-out start), pushing the total far above $106,506."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA expiration: it deducted uncapped SALT, including state income tax, and used pre-TCJA brackets up to 33%. PolicyEngine applies OBBBA's permanent TCJA brackets with the $32,200 standard + $1,070 non-itemizer charitable deduction. The frozen reference's $503,495 taxable income, $89,096 ordinary tax, $11,688 QD tax and $5,722 NIIT include a defective $2,164 IRA deduction and remain excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"Gave no derivation. Its $106,093 is $413 below the frozen reference's $89,095.92 ordinary tax + $11,688 QD tax + $5,721.98 NIIT, consistent with deductions about $1,290 above the frozen calculation at a 32% marginal rate. The frozen calculation includes a defective IRA deduction and remains excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of ESI premiums from wages (AGI $525,049), which PolicyEngine does not do, and took TCJA-sunset personal exemptions on top of itemized deductions, both of which understate taxable income. It also taxed qualified dividends at 20% even though taxable income is below the $613,700 threshold where 20% starts."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-expiration brackets, treated health insurance premiums as pre-tax wage reductions, and itemized with mortgage interest that isn't listed, producing a result below the frozen reference. The frozen calculation applies ordinary brackets to about $425,575 after removing $77,920 of qualified dividends from taxable income, producing $89,096 of ordinary tax plus $11,688 QD tax and $5,722 NIIT. It includes a defective IRA deduction and remains excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA sunset: it itemized state income and real estate taxes plus 2%-floor miscellaneous deductions for the $13,848 of employee expenses, which OBBBA permanently disallows. PolicyEngine uses the $32,200 standard + $1,070 charitable deduction. Its rounded $105,000 differs from the frozen reference's $89,095.92 + $11,688 + $5,721.98. The frozen calculation includes a defective IRA deduction and remains excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-4-argon,reference_engine_defect,taxable_income_or_deductions,False,"Applied post-TCJA-sunset 2026 rules instead of OBBBA's permanent TCJA brackets (24% to $403,550, then 32%) and the $32,200 standard + $1,070 charitable deduction. The frozen reference's $503,495 taxable income yields $106,506 including the $5,722 NIIT. These figures include a defective IRA deduction and remain excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"Left out the 3.8% NIIT ($5,722 on $150,578 of net investment income) included in the frozen reference; its inclusion in this output remains unresolved. It used a $30,000 standard deduction and 2025 brackets instead of 2026's $32,200 deduction and $211,400 22% ceiling, and omitted the $1,070 charitable deduction. Its inclusion of the $4,148 refund and full $46,308 traditional 401(k) deduction differ from the frozen calculation; those treatments remain unresolved."
-us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"Took a $2,000 non-itemizer charitable deduction, but OBBBA's §170(p) deduction covers only cash gifts, which total $1,070 here. Its $46,308 traditional 401(k) deduction differs from the frozen pro rata $41,650 allocation, whose treatment remains unresolved. Omitting the IRA deduction agrees with the recorded correction under 219(g). Its $500,070 taxable income is below the excluded frozen reference's $503,494.75, which includes the defective IRA deduction. It computed NIIT on $153,578 without the $3,000 capital-loss offset."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Reported $21,735 with no traceable calculation, about one-fifth of the frozen reference. The frozen calculation applies ordinary brackets to about $425,575 after removing $77,920 of qualified dividends from taxable income, producing $89,096 ordinary tax plus $11,688 QD tax and $5,722 NIIT. It includes a defective IRA deduction and remains excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"Cited itemized and medical deductions, but medical expenses fall far below the 7.5%-of-AGI floor, and the $32,200 standard + $1,070 charitable deduction beats itemizing. Its $75,657 is below the frozen reference's $89,096 ordinary-tax component, computed on about $425,575 after removing qualified dividends from taxable income. The frozen calculation includes a defective IRA deduction and remains excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its deductions ($32,200 + $1,070) and NIIT ($5,722) match the frozen components. Its $538,418 AGI is $1,653 above the frozen reference's $536,764.52. Omitting the IRA deduction agrees with the recorded correction under 219(g). Its refund inclusion and $46,308 traditional 401(k) deduction differ from the frozen refund exclusion and pro rata $41,650 allocation; those treatments remain unresolved. Its tax is $529 above the excluded frozen value."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Gave no figures. Its $104,535 is $1,971 below the excluded frozen reference of $106,505.90, consistent with taxable income about $6,160 below the frozen $503,494.75 at a 32% marginal rate. The frozen calculation includes a defective IRA deduction."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Included the $4,148 refund and itemized, differing from the frozen refund exclusion and $32,200 standard + $1,070 charitable deduction; refund taxation remains unresolved. PolicyEngine's SALT-cap phase-down and 0.5% AGI floor on itemized charity keep itemizing below those deductions. Its tax is $632 above the excluded frozen reference, whose taxable income includes a defective IRA deduction."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $32,200 standard deduction and omitted the $1,070 OBBBA non-itemizer cash charitable deduction. Its figure is consistent with $506,218 taxable income, $2,723 above the frozen reference's $503,494.75, and $871 more tax at 32%. Its inferred refund inclusion and $46,308 traditional 401(k) deduction differ from the frozen calculation; those treatments remain unresolved. Omitting the IRA deduction agrees with the recorded correction under 219(g), and the frozen value remains excluded."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted the $13,369 of employer-sponsored insurance premiums from wages (AGI $525,049), but PolicyEngine's wage income is $388,350 with no premium exclusion. It then itemized $36,959 instead of using the $33,270 standard + charitable deduction. Its taxable income is about $15,400 below the frozen reference's $503,494.75, accounting for a tax difference of about $4,930 at 32%. The frozen taxable income includes a defective IRA deduction, and the output remains excluded."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Included the $4,148 refund, which differs from the frozen refund exclusion under a refund treatment the record leaves unresolved, and described using the standard deduction without specifying its amount or treatment of the non-itemizer charitable deduction. Its tax is $571 above the excluded frozen reference, whose $503,494.75 taxable income includes a defective $2,164 IRA deduction."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Reduced wages by the $13,369 of ESI premiums (AGI $525,049), which PolicyEngine does not do, and itemized a phased-down SALT deduction instead of the $32,200 standard + $1,070 charitable deduction. Its tax is about $4,106 below the frozen reference, consistent with taxable income about $12,830 below the frozen $503,494.75 at a 32% marginal rate. The frozen taxable income includes a defective IRA deduction, and the output remains excluded."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"Reduced wages by the $13,369 of ESI premiums (AGI $525,049), which PolicyEngine does not do, and itemized $34,385.30 instead of using the $33,270 standard + charitable deduction. That gives $490,664 of taxable income, about $12,831 below the frozen reference's $503,494.75, accounting for a tax difference of $4,106 at 32%. The frozen taxable income includes a defective IRA deduction, and the output remains excluded."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6.1-terra,reference_engine_defect,taxable_income_or_deductions,False,"Itemized instead of using the $32,200 standard + $1,070 non-itemizer charitable deduction, and gave no traceable figures. Its answer is $1,094 below the frozen reference, consistent with deductions about $3,420 above the frozen calculation at a 32% marginal rate. The frozen $503,494.75 taxable income includes a defective IRA deduction, and the output remains excluded."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"Cited a QBI deduction even though the household has no business income, and gave no traceable calculation. Its $112,450 is $5,944 above the frozen reference's $89,095.92 + $11,688 + $5,721.98. The frozen calculation includes a defective IRA deduction and remains excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules (Pease, personal exemptions, 2% miscellaneous deductions) and then a $125,199 tentative AMT. Under 2026 OBBBA law none of those deduction rules apply, and AMT doesn't bind with the $140,200 MFJ exemption and $1,000,000 phase-out start. The frozen reference reports regular tax of $100,784 plus $5,722 NIIT. It includes a defective IRA deduction and remains excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"Applied TCJA-sunset rules: itemized deductions after Pease ($59,573) and a $25,486 AMT driven by SALT and miscellaneous add-backs. Under 2026 OBBBA law Pease and miscellaneous deductions are repealed, the $32,200 standard + $1,070 charitable deduction applies, and no AMT is due."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"Used the 2025 $31,500 standard deduction instead of 2026's $32,200 and omitted the $1,070 non-itemizer cash charitable deduction. Its $506,918 taxable income is above the excluded frozen reference's $503,494.75. Its refund inclusion and $46,308 traditional 401(k) deduction differ from the frozen calculation; those treatments remain unresolved. Omitting the IRA deduction agrees with the recorded correction under 219(g). Its $90,121 of ordinary tax is also about $70 off the 2026 schedule."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"Charged $106,958 of tax on $423,928 of ordinary income, where 2026 MFJ brackets give about $88,569. It also subtracted the $13,369 of ESI premiums from wages, claimed 2%-floor miscellaneous deductions that are permanently repealed, and computed NIIT on $153,578 without the $3,000 capital-loss offset."
-us,scenario_005,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"Assumed TCJA sunset: uncapped SALT (~$64,011), Pease, miscellaneous deductions, and an $18,221 AMT. It also subtracted ESI premiums from wages. PolicyEngine uses the $32,200 standard + $1,070 charitable deduction, and no AMT applies. The frozen reference's $536,764 AGI includes a defective IRA deduction and remains excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $106,505.90 reference."
-us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"Correctly chose the $32,200 standard deduction over itemizing, but left out the $1,070 non-itemizer cash charitable deduction. Its $506,218 taxable income is above the excluded frozen reference's $503,494.75, accounting for a tax difference of about $871 at 32%. Its refund inclusion and $46,308 traditional 401(k) deduction differ from the frozen calculation; those treatments remain unresolved. Omitting the IRA deduction agrees with the recorded correction under 219(g)."
-us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"Claimed deductions wipe out all tax, relying on invented mortgage interest on the $2.85M balance (no interest is listed) and on unreimbursed employee expenses that are not deductible. The frozen reference reports $503,495 of taxable income after the $32,200 standard + $1,070 charitable deduction, producing $106,505.90. These figures include a defective IRA deduction and remain excluded from scoring."
-us,scenario_005,federal_income_tax_before_refundable_credits,muse-spark-1.3,reference_engine_defect,taxable_income_or_deductions,False,"Left out the $5,722 NIIT included in the frozen reference; its inclusion in this output remains unresolved. It also used only the $32,200 standard deduction without the $1,070 charitable deduction. Its refund inclusion and full traditional 401(k) deduction give its stated $506,218 taxable income and differ from the frozen calculation; those treatments remain unresolved."
-us,scenario_005,federal_income_tax_before_refundable_credits,muse-wonder,reference_engine_defect,taxable_income_or_deductions,False,"Subtracted employer insurance premiums from wages (AGI near $525,000), which PolicyEngine does not do; its wage income is $388,350. Its tax is about $2,806 below the excluded frozen reference. The frozen $503,494.75 taxable income includes a defective IRA deduction."
-us,scenario_005,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"Took only the $32,200 standard deduction without the $1,070 charitable deduction. Its $506,218 taxable income is above the excluded frozen reference's $503,494.75. Its refund inclusion and $46,308 traditional 401(k) deduction differ from the frozen calculation; those treatments remain unresolved. Omitting the IRA deduction agrees with the recorded correction under 219(g). It also understated NIIT by using $142,667 of net investment income instead of $150,578 (interest $65,936 + dividends $87,642 − $3,000)."
-us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"Deducted the full $10,911 net capital loss in AGI instead of capping it at $3,000, and invented $43,118 of mortgage interest from an assumed 5.75% rate. It also netted the capital loss against qualified dividends, leaving only $67,009 at preferential rates, used a garbled bracket table, and omitted the $5,722 NIIT included in the frozen reference; the record leaves its inclusion in this output unresolved."
-us,scenario_005,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"Used the $15,700 single-filer standard deduction instead of the $32,200 MFJ amount, and understated gross income at $528,543 (pre-deferral income is about $585,000). Its $89,067 combined regular-plus-AMT figure is below the frozen reference's $89,096 ordinary tax, $11,688 qualified-dividend tax and $5,722 NIIT. The frozen calculation includes a defective IRA deduction, and NIIT inclusion in this output remains unresolved; the output stays excluded."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It left out the 3.8% net investment income tax ($5,721.98 on $150,578), which this variable includes, so its total stopped at regular tax. It also deducted the full $23,154 traditional 401(k) deferral per spouse instead of prorating to the $24,500 combined traditional-plus-Roth limit ($20,825 each). And it used a $32,600 standard deduction instead of $32,200 plus the $1,070 deduction for non-itemizers' cash gifts."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $23,154 traditional 401(k) per spouse without applying the $24,500 combined limit with the $4,086 Roth deferral (which prorates to $20,825), added the $4,148 state refund to income, and left out the $1,070 deduction for non-itemizers' cash gifts. Together these made taxable income $504,054 instead of $503,494.75, which is $179 too much tax at 32%."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It included the $4,148 state income tax refund in gross income. PolicyEngine excludes it, so the model's taxable income was $4,148 too high, adding $1,327 of tax at the 32% marginal rate. Every other step, including the prorated $20,825 deferrals, matched."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It invented about $110,000 of mortgage interest from the $2.85M balance even though no interest was listed. It then reported $249,914, a figure that no bracket calculation on its own stated taxable income produces. The correct path is the $32,200 standard deduction plus $1,070 for cash gifts, giving $89,096 ordinary tax + $11,688 on qualified dividends + $5,722 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It said it did not compute the liability and entered 0 as a placeholder. Taxable income of $503,495 produces $89,096 of ordinary tax, plus $11,688 on qualified dividends and $5,722 of NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly left out NIIT ($5,722), which this variable includes. It also computed regular tax of $102,907 but submitted $91,458, dropping the $11,688 qualified-dividend tax from its own total. On top of that it used a $31,500 standard deduction and added the $4,148 state refund to income."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It invented mortgage interest from the loan balance to itemize about $60,000 under a $10,000 SALT cap, when the standard deduction plus $1,070 for cash gifts applies. It taxed qualified dividends at 20% even though taxable income is below the 20% threshold. Its final $154,430 does not even follow from its own parts (about $92,000 + $15,584)."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It made an arithmetic error on AGI, reporting $604,318 when its own components add up to $538,418. It also invented mortgage interest from the loan balance and applied a made-up 18.8% rate to qualified dividends, which pushed the result to $141,000."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $23,154 traditional 401(k) per spouse without applying the $24,500 combined traditional-plus-Roth limit ($20,825 each), added the $4,148 state refund to income, and left out the $1,070 deduction for non-itemizers' cash gifts. That gave taxable income of $504,054 instead of $503,494.75, which is $179 too much tax."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and used pre-2018 brackets topping out in the 33% range, restored 2% miscellaneous deductions, and added $7,141 of AMT, but OBBBA made the TCJA rate structure permanent. It also invented $52,500 of mortgage interest at an assumed 7% rate and computed NIIT on $153,578 without subtracting the $3,000 capital loss."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It invented about $28,000 of mortgage interest under a $10,000 SALT cap to itemize $40,574, when the $32,200 standard deduction plus $1,070 for cash gifts applies. It never ran the bracket calculation and submitted a rounded $115,000 instead of $89,096 ordinary + $11,688 qualified-dividend tax + $5,722 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It added the $4,148 state refund to AGI, deducted the full $23,154 deferral per spouse instead of the prorated $20,825, and itemized about $34.5k instead of taking the $32,200 standard deduction plus $1,070 for cash gifts. It then rounded its total to $106.65k."
+us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It used a $30,750 standard deduction and wrong bracket thresholds, getting $92,234 of tax on $429,748 of ordinary income, when the 2026 schedule gives about $90,431. It also added the $4,148 refund to income and computed NIIT on $153,578 without subtracting the $3,000 capital loss ($5,836 instead of $5,722)."
+us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It imposed a $131,247 AMT that does not apply. Under the 2026 OBBBA AMT exemption, tentative minimum tax stays below regular tax for a household taking the standard deduction. Its regular-tax base of $465,449 also implies about $70k of made-up itemized deductions."
+us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It applied the Pease limit to about $59.5k of itemized deductions that include made-up items, then imposed AMT, both of which assume the TCJA expired. Under 2026 law the household takes the $32,200 standard deduction plus $1,070 for cash gifts and owes no AMT. It also computed NIIT on $153,578 instead of $150,578."
+us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It taxed ordinary income under pre-TCJA 2026 brackets and used $63,508 of made-up itemized deductions, but OBBBA made the TCJA brackets permanent and the household takes the standard deduction. It also computed NIIT on $153,578 without the $3,000 capital loss."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and used uncapped SALT including made-up state income tax, mortgage interest on $1M of acquisition debt, and $10,400 of personal exemptions, for $108,283 of itemized deductions. OBBBA kept the standard-deduction regime, so taxable income is $503,495, not the much lower base that produced its $88,016 of ordinary tax."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It built an AGI of $460,936 that counts the tax-exempt $70,000 pension and leaves out much of the dividend and interest income, instead of $536,765. It also never broke out the 15% qualified-dividend tax or NIIT that make up the $106,506 total."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $13,369 of employer-sponsored insurance premiums from wages, giving AGI of $525,049, although taxable wages are $388,350 after only the prorated 401(k) deferrals. It then added AMT, which does not apply under the 2026 OBBBA AMT exemption."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026, deducting full SALT including made-up state income tax and using pre-2018 brackets up to 33%. OBBBA kept the 2026 TCJA brackets and the $32,200 standard deduction, which together with the $1,070 for cash gifts gives $89,096 of ordinary tax."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation, and its figure is $413 below the reference, which equals about $1,290 too little taxable income at 32%. It did not reproduce PolicyEngine's taxable income of $503,494.75, which rests on prorated $20,825 deferrals, the state refund left out, and the $32,200 standard deduction plus $1,070 for cash gifts."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted employer health premiums from wages (AGI $525,049) and assumed the TCJA expired, using itemized deductions and personal exemptions. It also taxed qualified dividends at 20% instead of 15%, even though taxable income is below the 20% threshold."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used brackets that assume the TCJA expired, subtracted the health insurance premiums from wages, and itemized made-up mortgage interest, which drove taxable income and tax well below the reference. The correct path is the OBBBA 2026 brackets applied to $503,495 of taxable income after the $32,200 standard deduction plus $1,070."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It itemized state income tax and 2% miscellaneous expenses on the assumption that the TCJA expired, but OBBBA permanently eliminated miscellaneous itemized deductions and the household takes the $32,200 standard deduction plus $1,070 for cash gifts. It also deducted the full $23,154 deferrals instead of the prorated $20,825, then rounded to $105,000."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-4-argon,reference_engine_defect,taxable_income_or_deductions,False,"It applied rules that assume the TCJA expired, with pre-2018 brackets that put ordinary income in the 33% bracket and a different deduction regime. Ordinary tax is $89,095.92 under the permanent OBBBA 2026 joint-filer brackets on $425,575."
+us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $5,722 NIIT that this variable includes. It also used 2025 brackets and a $30,000 standard deduction instead of the 2026 $32,200 plus the $1,070 deduction for non-itemizers' cash gifts, and it added the $4,148 refund to income."
+us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It took the full $2,000 non-itemizer charitable cap even though cash gifts were only $1,070, and the deduction is limited to actual cash contributions. It also deducted the full $23,154 deferral per spouse instead of the prorated $20,825, and computed NIIT on $153,578 without the $3,000 capital loss, leaving taxable income at $500,070 instead of $503,495."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It reported $21,735 without any bracket calculation, far below what about $503k of taxable income produces. The correct total is $89,096 of ordinary tax plus $11,688 on qualified dividends plus $5,722 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It claimed itemized deductions for real estate taxes and medical costs, even though medical costs are far below the 7.5%-of-AGI floor and the standard deduction applies, and it left out NIIT. Its $75,657 is about $31k too low compared with $89,096 + $11,688 + $5,722."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It added the $4,148 state refund to income and deducted the full $23,154 traditional 401(k) per spouse instead of prorating to the $24,500 combined limit ($20,825 each). It also took no IRA deduction, while PolicyEngine deducts $2,164. Taxable income came to $505,148 instead of $503,494.75, which is $529 too much tax."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It gave no figures, and its $104,535 is $1,971 below the reference, which equals about $6,200 too little taxable income at 32%. It did not reproduce $503,494.75 of taxable income: wages of $388,350 after prorated deferrals, the refund left out, and the $32,200 standard deduction plus $1,070."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It added the $4,148 state refund to taxable income and itemized instead of taking the $32,200 standard deduction plus the $1,070 deduction for non-itemizers' cash gifts. Both put taxable income above $503,494.75, producing a $632 overstatement."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It added the $4,148 state refund to AGI, deducted the full $23,154 deferral per spouse instead of the prorated $20,825, and took only the $32,200 standard deduction without the $1,070 deduction for non-itemizers' cash gifts. Taxable income came to about $506,218 instead of $503,494.75, which is $871 too much tax."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $13,369 of employer-sponsored insurance premiums from wages, giving AGI of $525,049 instead of $536,765. It also itemized $36,959.30 instead of taking the $32,200 standard deduction plus $1,070 for cash gifts, which understated taxable income."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It added the $4,148 state refund to income, deducted the full $23,154 deferrals instead of the prorated $20,825, and left out the $1,070 deduction for non-itemizers' cash gifts. Its taxable income ended up about $1,780 above $503,494.75, which is $571 too much tax."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted employer health premiums from wages, giving AGI of $525,049 instead of $536,765, and itemized a phased-down SALT deduction instead of taking the $32,200 standard deduction plus $1,070. Taxable income came out about $12.9k too low."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $13,369 of employer-sponsored insurance premiums from wages (AGI $525,049) and itemized $34,385.30 instead of taking the $32,200 standard deduction plus $1,070. Together these put taxable income about $12.9k below $503,494.75."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-6.1-terra,reference_engine_defect,taxable_income_or_deductions,False,"It itemized instead of taking the $32,200 standard deduction plus $1,070 for cash gifts, and ended up $1,094 low, which equals about $3,400 too little taxable income at 32%. PolicyEngine's taxable income is $503,494.75."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It relied on a QBI deduction and mortgage interest, even though the household has no business income and no interest was listed, and it never computed NIIT separately. Its $112,450 is $5,944 above the correct $89,096 ordinary + $11,688 qualified-dividend + $5,722 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied rules that assume the TCJA expired: the Pease limit, personal exemptions, and miscellaneous itemized deductions. It then imposed a $125,199 tentative AMT. Under 2026 OBBBA law the household takes the standard deduction and owes no AMT."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It used rules that assume the TCJA expired, with $59,573 of itemized deductions after the Pease limit, and added $25,486 of AMT from SALT and miscellaneous add-backs. Under 2026 law no Pease or AMT applies and the $32,200 standard deduction plus $1,070 is used."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It used the $31,500 standard deduction instead of the 2026 $32,200 plus the $1,070 deduction for non-itemizers' cash gifts. It also added the $4,148 state refund to income and deducted the full $23,154 deferrals instead of the prorated $20,825, leaving taxable income at $506,918."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted employer health premiums from wages and itemized uncapped property tax plus 2% miscellaneous expenses, on the assumption that the TCJA expired. It then charged $106,958 on $423,928 of ordinary income when the 2026 brackets give about $88,600, and computed NIIT on $153,578 instead of $150,578."
+us,scenario_005,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It applied rules that assume the TCJA expired: uncapped SALT including about $64k of made-up state income tax, miscellaneous deductions, and the Pease limit. It added $18,221 of AMT and subtracted employer health premiums from wages, but 2026 law uses the standard deduction with no AMT."
+us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this variable, so there is no answer to score."
+us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,reference_engine_defect,taxable_income_or_deductions,False,"It added the $4,148 state refund to AGI, deducted the full $23,154 deferral per spouse instead of the prorated $20,825, and took only the $32,200 standard deduction without the $1,070 deduction for non-itemizers' cash gifts. Taxable income came to $506,218 instead of $503,494.75, an $871 overstatement."
+us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It claimed that made-up mortgage interest on the $2.85M balance and other deductions wipe out all taxable income. In fact no interest was listed, the standard deduction applies, and taxable income is $503,494.75, which carries $106,506 of tax."
+us,scenario_005,federal_income_tax_before_refundable_credits,muse-spark-1.3,reference_engine_defect,taxable_income_or_deductions,False,"It left out the $5,722 NIIT that this variable includes. It also added the $4,148 refund, deducted the full deferrals, and skipped the $1,070 deduction for non-itemizers' cash gifts, which put taxable income at $506,218."
+us,scenario_005,federal_income_tax_before_refundable_credits,muse-wonder,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted employer-sponsored insurance premiums from wages, giving AGI of about $525,000 instead of $536,765. That understated taxable income by roughly $12k compared with PolicyEngine's $503,494.75."
+us,scenario_005,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It computed NIIT on $142,667 instead of $150,578 of net investment income ($5,421 instead of $5,722). It also added the $4,148 refund, deducted the full $23,154 deferrals, and skipped the $1,070 deduction for non-itemizers' cash gifts, leaving taxable income at $506,218."
+us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $10,911 net capital loss instead of the $3,000 cap, netted the loss against qualified dividends (taxing $67,009 at 15% instead of $77,920), and itemized made-up mortgage interest at 5.75% under a $10,000 SALT cap. It also used wrong bracket thresholds and left out NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It used a $15,700 single-filer standard deduction instead of the $32,200 joint-filer amount plus $1,070, and understated gross income at $528,543. It also added AMT and left out NIIT, so it missed the $11,688 qualified-dividend plus $5,722 NIIT structure."
us,scenario_005,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_005,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model used a $183,600 Social Security wage base instead of $184,500, and a 1.2% SDI rate instead of the 2026 rate of 1.3% on all $430,000 ($5,590). It reached $35,558.20 in its reasoning but then submitted an unrelated $33,296.55."
-us,scenario_005,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"The model's federal FICA figure ($30,454) was correct, but it left out California's mandatory employee SDI, which is 1.3% of $430,000 = $5,590."
-us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model used the 2024 Social Security cap of $168,600 instead of $184,500. It also applied a 1% SDI rate with a $153,164 wage cap, even though the cap was repealed in 2024 and the 2026 rate is 1.3% uncapped. On top of that, it added roughly $10,087 of state income tax into the payroll tax total."
-us,scenario_005,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"The model never computed anything and submitted 0. Applying FICA to $430,000 of wages ($30,454), plus 1.3% CA SDI ($5,590), gives $36,044."
-us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"The model capped the spouse's Social Security wages at $181,800 instead of the 2026 base of $184,500. It also applied CA SDI at 1.2% instead of 1.3%, so SDI came out at $5,160 instead of $5,590."
-us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model correctly got $30,454 of federal FICA, then left out CA SDI (1.3% × $430,000 = $5,590) because it was unsure of the rate. It then submitted $28,586.60, a number that doesn't match its own figures."
-us,scenario_005,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model said its total included CA SDI, but its $30,437 contains no SDI at all; it is essentially the federal FICA sum of $30,454, minus a $17 arithmetic error. It omitted the 1.3% uncapped SDI of $5,590."
-us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the 2025 wage base of $176,100 instead of $184,500, which wrongly capped the head's $180,000 below their full wages and also understated the spouse's capped amount. It also applied CA SDI at 1.2% instead of 1.3%."
-us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model used a $176,100 Social Security cap for both earners (it should be $184,500, and the head's $180,000 falls below it) and a 1.1% SDI rate instead of 1.3%. It then discarded its own $34,421.40 and submitted an unsupported $27,306."
-us,scenario_005,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"The model calculated Social Security for only one capped wage base ($11,253), leaving out the second earner's Social Security tax, even though each worker has their own cap. It also omitted CA SDI of 1.3% × $430,000 = $5,590."
-us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"The model understated Social Security ($22,407 instead of $22,599) by using a cap below $184,500. It put CA SDI at $4,080, which is far below the correct 1.3% × $430,000 = $5,590."
-us,scenario_005,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model applied CA SDI at 1.2% subject to a $153,164 per-worker cap, but that cap was repealed starting in 2024. The 2026 SDI is 1.3% of all $430,000 ($5,590). The model also used a $181,800 Social Security cap instead of $184,500."
-us,scenario_005,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590). It also capped Social Security at $184,200 instead of $184,500."
-us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model computed CA SDI at 1.1% ($4,730) instead of 1.3% ($5,590). Its Social Security figure of $21,242 implies the head's $180,000 was wrongly capped. The correct amount is 6.2% × ($180,000 + $184,500) = $22,599."
-us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model counted only federal Social Security and Medicare and left out CA SDI ($5,590 at 1.3% uncapped). Even its federal figure of $31,398 doesn't match the correct FICA sum of $30,454."
-us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted employer-sponsored insurance premiums from wages before computing FICA and SDI. The payroll tax base is the full $430,000 of gross wages, so this shortchanged both, and its SDI also falls short of 1.3% × $430,000 = $5,590."
-us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model applied CA SDI at 1.1% instead of the 2026 rate of 1.3%, losing $860. Its total also implies Social Security below $22,599, the correct figure using the $184,500 cap on the spouse."
-us,scenario_005,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model left out CA SDI ($5,590), and its $26,588 also understates federal FICA. The correct federal amount is $22,599 Social Security + $6,235 Medicare + $1,620 Additional Medicare = $30,454."
-us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model subtracted ESI premiums from both earners' wages (giving $173,111 and $243,520), shrinking the FICA, Additional Medicare and SDI bases below the full $430,000. It also used a 1.1% SDI rate instead of 1.3%, and a $181,200 Social Security cap instead of $184,500."
-us,scenario_005,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"The model's $33,814 means it counted only about $3,360 above the $30,454 federal FICA. The correct CA SDI is 1.3% × $430,000 = $5,590, so its SDI was understated by an outdated rate or a repealed cap."
-us,scenario_005,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"The model left out CA SDI ($5,590) entirely. It also capped both earners at the old $176,100 base, though the head's $180,000 falls entirely below the 2026 cap of $184,500."
-us,scenario_005,payroll_tax,gemini-4-argon,llm_error,state_local_rule,False,"The model applied CA SDI at 1.1% ($4,730) instead of 1.3% ($5,590), and capped the spouse at $181,800 instead of the 2026 base of $184,500."
-us,scenario_005,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"The model applied CA SDI with a $165,384 per-worker wage cap, but SB 951 removed that cap starting in 2024. The correct SDI is 1.3% of all $430,000 = $5,590. The model also used a $182,100 Social Security cap instead of $184,500."
-us,scenario_005,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"The model wrongly said California has no mandatory employee payroll tax, which led it to leave out the 1.3% uncapped SDI contribution of $5,590. Its federal FICA of $30,454 was correct."
-us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model wrongly concluded that no Additional Medicare Tax applies, even though combined wages of $430,000 exceed the $250,000 MFJ threshold by $180,000 ($1,620). It also treated both spouses as above the Social Security cap when the head is below it, and it left out CA SDI ($5,590)."
-us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's $13,905 is less than even the Social Security tax alone ($22,599), which means it effectively capped Medicare and left out large components. The correct total is $30,454 of federal FICA plus $5,590 of 1.3% CA SDI."
-us,scenario_005,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"The model applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590). It also capped the spouse's Social Security wages at $183,600 instead of $184,500."
-us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model used roughly 1.3% for CA SDI ($5,590), but its $36,667 total overshoots by $623. That means its federal components came out above the correct $30,454 ($22,599 Social Security with the $184,500 cap + $6,235 Medicare + $1,620 Additional Medicare)."
-us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"The model's $35,614 is exactly $30,454 of FICA plus SDI at 1.2% ($5,160). The 2026 CA SDI rate is 1.3% on uncapped wages, which gives $5,590."
-us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The model's $35,614 is exactly $30,454 of FICA plus SDI at 1.2% ($5,160). The 2026 CA SDI rate is 1.3% on all $430,000, which gives $5,590."
-us,scenario_005,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model used the correct 1.3% SDI rate and $184,500 cap, but it subtracted $13,369 of employer-sponsored insurance premiums from wages as cafeteria-plan reductions. The full $430,000 of gross wages is the base for FICA, Additional Medicare and SDI."
-us,scenario_005,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"The model computed federal FICA correctly at $30,454, but applied CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% ($5,590)."
-us,scenario_005,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model used the correct 1.3% SDI rate and $184,500 cap, but treated ESI premiums as pretax and reduced wages to $416,631. Gross wages of $430,000 are the base for FICA, Additional Medicare and SDI."
-us,scenario_005,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"The model's reasoning covers only OASDI, HI and Additional Medicare, which total $30,454, and it never computed CA SDI at 1.3% of $430,000 ($5,590). Its $34,515 understates the correct sum by $1,529."
-us,scenario_005,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model applied CA SDI at 1.2% with a wage cap ($4,226), but the cap was repealed in 2024 and the 2026 rate is 1.3% on all $430,000 ($5,590). It also capped the spouse's Social Security wages at about $182,000 instead of $184,500."
-us,scenario_005,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"The model applied CA SDI at 1.2% subject to an SDI wage limit, but no SDI wage limit exists after 2024. The 2026 rate is 1.3% of the full $430,000, which is $5,590."
-us,scenario_005,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted $13,369 of ESI premiums from the FICA wage base and applied SDI at 1.1% to an invented $83,067 per-person cap. The correct SDI is 1.3% of the full $430,000 with no cap, and FICA is based on gross wages."
-us,scenario_005,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model subtracted ESI premiums from gross wages and capped Social Security at the stale $168,600 base instead of $184,500. It also applied SDI at about 1.2% to capped wages, when the correct SDI is 1.3% of all $430,000."
-us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no payroll_tax value and no explanation, so there was no answer to score."
-us,scenario_005,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"The model got federal FICA right at $30,454 and correctly treated SDI as uncapped, but used 1.2% ($5,160) instead of the 2026 CA SDI rate of 1.3% ($5,590)."
-us,scenario_005,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model wrongly claimed that income tax deductions offset employee payroll taxes, and submitted 0. Payroll taxes apply to gross wages with no such offset: $22,599 Social Security + $6,235 Medicare + $1,620 Additional Medicare + $5,590 CA SDI."
-us,scenario_005,payroll_tax,muse-spark-1.3,llm_error,state_local_rule,False,"The model got federal FICA right at $30,454, but computed CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)."
-us,scenario_005,payroll_tax,muse-wonder,llm_error,payroll_tax_base,False,"The model subtracted cafeteria-plan premiums from each spouse's wages before applying Social Security, Medicare and SDI. The base is full gross wages of $430,000, with SDI at 1.3% ($5,590)."
-us,scenario_005,payroll_tax,ox-alpha,llm_error,thresholds_rates,False,"The model applied the Additional Medicare Tax to the spouse's wages over the $200,000 single-filer withholding threshold ($2,250). For MFJ, the tax applies to combined wages over $250,000, giving $1,620. The model also left out CA SDI of 1.3% × $430,000 = $5,590."
-us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model applied CA SDI at 1.1% subject to a $153,164 per-worker cap, but the cap was repealed in 2024 and the 2026 rate is 1.3% uncapped ($5,590). It also capped both earners at the 2025 base of $176,100 instead of $184,500, which wrongly limited the head's $180,000."
-us,scenario_005,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"The model left out CA SDI (1.3% × $430,000 = $5,590). Its federal FICA of $25,474 also falls short of the correct $30,454 ($22,599 Social Security + $6,235 Medicare + $1,620 Additional Medicare)."
+us,scenario_005,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It used CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on all $430,000 ($5,590), and a $183,600 Social Security base instead of $184,500. It then submitted $33,296.55, which does not match its own $35,558.20 sum."
+us,scenario_005,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Its federal FICA of $30,454 is correct, but it left out California's mandatory employee SDI of 1.3% on all $430,000 in wages, which is $5,590."
+us,scenario_005,payroll_tax,claude-fable-5.2,llm_error,payroll_tax_base,False,"It took traditional 401(k) deferrals out of the payroll wage base, but elective deferrals still count as FICA and SDI wages, so the full $430,000 is taxable. This understated Medicare, Additional Medicare, and SDI (which it correctly put at 1.3%)."
+us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 Social Security base of $168,600 and capped the head's $180,000 wrongly. It also applied a 1% SDI rate with a $153,164 cap, when 2026 SDI is 1.3% uncapped. On top of that it added ETT and $10,086.52 of state income tax, which are not employee payroll taxes, and ended at $41,859.60."
+us,scenario_005,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"It skipped the calculation and entered 0. The correct sum is $22,599 Social Security, $6,235 Medicare, $1,620 Additional Medicare, and $5,590 CA SDI, totaling $36,044."
+us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It used CA SDI at 1.2% instead of 1.3% (missing $430) and a $181,800 Social Security base instead of $184,500 (missing $167.40). That accounts for the $597.40 shortfall exactly."
+us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It deliberately left out CA SDI, the mandatory employee tax of 1.3% on all wages ($5,590), and then submitted $28,586.60, which is lower than its own $30,454 FICA subtotal."
+us,scenario_005,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It claimed CA SDI was already included, but its $30,437 is just FICA (and slightly below its own $30,454 subtotal). The required 1.3% SDI on $430,000, or $5,590, was never added."
+us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 Social Security base of $176,100 instead of $184,500 and wrongly capped the head's $180,000 (short $762.60). It also used SDI at 1.2% instead of 1.3% (short $430)."
+us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"It capped both earners at a $176,100 Social Security base, although the head's $180,000 sits under the 2026 base of $184,500, and it used SDI at 1.1% instead of 1.3%. It then submitted $27,306, which matches neither of its own subtotals ($29,691.40 or $34,421.40)."
+us,scenario_005,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It applied a single Social Security cap ($11,253) to the whole household instead of capping each worker separately, which left out about $11,300 of tax. It also left out CA SDI of $5,590."
+us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Its CA SDI of $4,080 is too low; the 2026 rule is 1.3% of all $430,000 with no cap, or $5,590. Its Social Security base was also about $181,400 instead of $184,500, giving $22,407 instead of $22,599."
+us,scenario_005,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It capped SDI at $153,164 per worker at 1.2%, but California removed the SDI wage ceiling in 2024, so 2026 SDI is 1.3% of all $430,000 ($5,590). It also used a Social Security base of about $181,800 instead of $184,500."
+us,scenario_005,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It used CA SDI at 1.2% ($5,160) instead of 1.3% ($5,590) and a $184,200 Social Security base instead of $184,500."
+us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used SDI at 1.1% ($4,730) instead of 1.3% ($5,590). Its Social Security figure of $21,242 implies both earners were capped near $171,300, when the head's $180,000 falls under the $184,500 base and should be fully taxed."
+us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It counted only federal FICA and left out California's mandatory 1.3% employee SDI on $430,000 ($5,590). Its FICA total also differs from the correct $30,454."
+us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It subtracted employer-sponsored insurance premiums from the payroll wage base. The reference taxes the full gross $430,000, so Medicare, Additional Medicare, and SDI came out too low."
+us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It used CA SDI at 1.1% instead of the 2026 rate of 1.3% on all $430,000 ($5,590). Its Social Security figure also implies both earners were capped below $184,500, so the head's $180,000 was not fully taxed."
+us,scenario_005,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It included only Social Security and Medicare and left out CA SDI ($5,590). Its $26,588 total also falls short of the correct FICA-plus-Additional-Medicare figure of $30,454."
+us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It took ESI premiums out of each person's wages (head down to $173,111), but the reference applies payroll taxes to the full $430,000. It also used SDI at 1.1% instead of 1.3%."
+us,scenario_005,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $33,814 is $2,230 short of the correct $22,599 Social Security + $6,235 Medicare + $1,620 Additional Medicare + $5,590 SDI. That shortfall fits SDI taken below 1.3% of all wages, along with a Social Security base below $184,500."
+us,scenario_005,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It left out CA SDI ($5,590). It also capped both earners at $176,100, although the head's $180,000 is under the 2026 base of $184,500."
+us,scenario_005,payroll_tax,gemini-4-argon,llm_error,state_local_rule,False,"It used CA SDI at 1.1% ($4,730) instead of 1.3% ($5,590) and a $181,800 Social Security base instead of $184,500."
+us,scenario_005,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It capped SDI at $165,384 per worker at 1.2%, but California eliminated the SDI wage cap in 2024, and the 2026 rule is 1.3% of all $430,000 ($5,590). Its Social Security base of $182,100 was also below $184,500."
+us,scenario_005,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It stated that California has no mandatory employee state payroll tax, which skips CA SDI at 1.3% of all wages ($5,590). Its FICA of $30,454 is correct."
+us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"It said the Additional Medicare Tax doesn't apply, but combined wages of $430,000 exceed the $250,000 joint threshold by $180,000, producing $1,620. It also left out CA SDI ($5,590) and understated Social Security and Medicare."
+us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"Its $13,905 is roughly one worker's FICA. The correct total taxes both earners separately ($22,599 Social Security and $6,235 Medicare) and adds $1,620 Additional Medicare and $5,590 CA SDI."
+us,scenario_005,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It used CA SDI at 1.2% ($5,160) instead of 1.3% ($5,590) and a $183,600 Social Security base instead of $184,500."
+us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"It overshot the correct total by $623, which fits a spouse Social Security cap set about $10,000 above the actual 2026 base of $184,500. Social Security should be exactly 6.2% of $364,500, or $22,599, alongside $6,235 + $1,620 + $5,590."
+us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"Its FICA of $30,454 is correct, but it used CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)."
+us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Its FICA of $30,454 is correct, but it used CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)."
+us,scenario_005,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It treated ESI premiums as pre-tax and took them out of the FICA and SDI wage base. The reference applies payroll taxes to the full gross $430,000, so Social Security, Medicare, Additional Medicare ($1,499.68 instead of $1,620), and SDI all came out too low, although its 1.3% SDI rate is correct."
+us,scenario_005,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"Its FICA of $30,454 is correct, but it used CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)."
+us,scenario_005,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated employer insurance premiums as pre-tax and took them out of payroll wages. The reference taxes the full $430,000 gross, so every component came out too low despite the correct 1.3% SDI rate and $184,500 base."
+us,scenario_005,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Its reasoning lists only federal OASDI, HI, and Additional Medicare and never applies CA SDI at 1.3% of all $430,000 ($5,590). The $34,515 total is $1,529 short of the correct amount."
+us,scenario_005,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It applied a wage cap to CA SDI and got $4,226, but the cap was eliminated in 2024, and 2026 SDI is 1.3% of all $430,000 ($5,590). Its spouse Social Security base was also below $184,500."
+us,scenario_005,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It applied 1.2% SDI up to an SDI wage limit, but California SDI no longer has a wage cap, and the 2026 rate is 1.3% of all $430,000 ($5,590)."
+us,scenario_005,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It took $13,369 of health premiums out of FICA wages, but the reference taxes the full $430,000. It also applied SDI at 1.1% on a made-up $83,067 per-person base instead of 1.3% uncapped ($5,590)."
+us,scenario_005,payroll_tax,inkling,llm_error,payroll_tax_base,False,"It took health premiums out of FICA wages and used the outdated $168,600 Social Security base. It also capped CA SDI, when the correct 2026 SDI is 1.3% of all $430,000 gross wages."
+us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was no answer to score."
+us,scenario_005,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"Its FICA of $30,454 is correct, but it used CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)."
+us,scenario_005,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"It claimed that deductions offset employee payroll taxes, but FICA and SDI are flat levies on wages with no income-tax offsets. The household owes $36,044."
+us,scenario_005,payroll_tax,muse-spark-1.3,llm_error,state_local_rule,False,"Its FICA of $30,454 is correct, but it used CA SDI at 1.2% ($5,160) instead of the 2026 rate of 1.3% on $430,000 ($5,590)."
+us,scenario_005,payroll_tax,muse-wonder,llm_error,payroll_tax_base,False,"It took cafeteria-plan health premiums out of each spouse's payroll wages, but the reference applies Social Security, Medicare, Additional Medicare, and SDI to the full $430,000 gross."
+us,scenario_005,payroll_tax,ox-alpha,llm_error,thresholds_rates,False,"It figured the Additional Medicare Tax as $2,250 from the spouse's wages alone. The correct figure is 0.9% of combined wages above the $250,000 joint threshold, or $1,620. It also left out CA SDI ($5,590)."
+us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It capped SDI at $153,164 per worker at 1.1%, when 2026 SDI is 1.3% of all wages with no cap ($5,590). It also used the $176,100 Social Security base, which wrongly capped the head's $180,000."
+us,scenario_005,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It left out CA SDI ($5,590), and its federal components total only $25,474 instead of the correct $22,599 Social Security + $6,235 Medicare + $1,620 Additional Medicare = $30,454."
us,scenario_005,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It deducted about $22,800 of mortgage interest on the $2.85M balance even though no interest amount was given, so that input is 0. It also assumed the exemption credit fully phases out (the engine keeps $114). Its own taxable income of $490,509 gives about $38,800, yet it reported $45,856 based on an unexplained 'refined bracket computation' that does not follow from its own numbers."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $532,106 is $6,822 below the engine's $538,928.31. It is also $2,164 below its own federal-AGI-minus-refund figure, so it effectively deducted the traditional IRA contributions, which are nondeductible for active plan participants at this income. Taxable income of $510,546 (vs. $518,408) plus a $210 exemption credit (vs. $114) left the answer about $825 low."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted federal income tax paid and employer insurance premiums, neither of which California allows. It also applied a nonexistent 12% bracket above $325,000. Finally, it invented about $21,621 of nonrefundable credits, including a 'standard deduction credit', when the only credit is the phased-down $114 exemption credit."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It skipped the calculation and entered 0. The correct path is CA AGI of $538,928 minus $20,520 of limited itemized deductions, which gives $41,089 of tax less the $114 exemption credit."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It deducted about $60,000 of mortgage interest it estimated from the $2.85M balance, even though unlisted interest counts as 0. It also left out the $2,986 deduction for employee expenses above the 2% floor and the high-income itemized cutback. Its taxable income was about $65k too low. It also took $560 of exemption credits; the engine allows $114 after the phaseout."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension to CA income and deducted about $35,000 of mortgage interest that was never listed. The $36,767 it reported does not follow from the 9.3% schedule applied to its own $595,000 taxable income, which would give about $48,000."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It put CA taxable income at about $560,000 instead of $518,408, treating the tax-exempt pension as partly taxable and not building the itemized deduction correctly. It also described rates rising to 11.3%, even though this income stays in the 9.3% bracket. Together these overstated the tax by about $6,500."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $532,106 effectively subtracts the $2,164 of nondeductible traditional IRA contributions and is $6,822 below the engine's $538,928.31. That left taxable income at $509,652 instead of $518,408. It also used a $256 exemption credit instead of the $114 left after the phaseout."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used CA AGI of $534,270, which is $4,658 below the engine's $538,928.31. It did not apply California's high-income itemized limitation (a $2,319.90 cutback), so it deducted $23,017 instead of $20,520. It then subtracted a $500 exemption credit with no phaseout instead of $114, which pulled the result about $790 low."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It deducted an assumed $50,000 of mortgage interest even though no interest was listed, and it left out the employee-expense deduction above the 2% floor. Its own brackets applied to $464,416 give about $36,700, yet it reported $32,000, so its arithmetic is wrong as well."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used CA AGI of about $534k instead of $538,928 and itemized only property tax and charity ($19.9k), leaving out the $2,986 of employee expenses above the 2% floor and the high-income cutback. It also treated the exemption credit as fully phased out, when $114 remains."
-us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Its taxable income of $518,564 is nearly the engine's $518,408. However, it applied the CA MFJ schedule incorrectly and got $42,151 instead of about $41,089. It also claimed no nonrefundable credits apply, ignoring the $114 exemption credit that survives the phaseout."
-us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It understated CA taxable income at $497,700 instead of $518,408, about $20.7k too much in deductions or too little AGI. It also subtracted a $300 exemption credit instead of the phased-down $114."
-us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its taxable income of $512,889 is about $5.5k below the engine's $518,408, reflecting a CA AGI below $538,928.31. It also stated that no nonrefundable credits apply, but the engine subtracts a $114 exemption credit."
-us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It started from CA AGI of $534,270, which is $4,658 below the engine's $538,928.31, and itemized $18,663 instead of $20,519.93, giving taxable income of $515,607 instead of $518,408. It also used a $310 exemption credit instead of the phased-down $114."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $73,017 of itemized deductions includes about $50,000 of mortgage interest that was never listed and counts as 0. That cut taxable income to $461,253 instead of $518,408. It also used CA AGI of $534,270 instead of $538,928 and a $288 exemption credit instead of $114."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path is CA AGI of $538,928 minus $20,520 of limited itemized deductions, taxed at the 9.3% top bracket, less the $114 credit, for $40,975. Its $31,215 is about $9,760 low, which matches about $105k of extra deductions, such as deducting assumed mortgage interest on the $2.85M balance."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its taxable income of $501,848 is $16,560 below the engine's $518,408. That figure matches subtracting the $13,369 of employer-sponsored insurance premiums from wages and skipping the high-income itemized cutback. The engine does neither."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $520,901 equals $534,270 minus the $13,369 of employer-sponsored insurance premiums. The engine does not subtract those premiums and has CA AGI of $538,928.31. It also itemized only $19,854, leaving out the $2,986 of employee expenses above the 2% floor and the high-income cutback."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path is $518,408 of taxable income, $41,089 of tax and a $114 exemption credit. Its $33,816 is about $7,160 low, which matches understating taxable income by roughly $77k, through deductions that are not listed or too little AGI."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It began from federal AGI without the SALT-refund subtraction, which happened to land its taxable income close to the engine's $518,408. It then applied the CA schedule incorrectly and got about $42,150 instead of $41,089. It also left out the $114 exemption credit."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted 'pre-tax deductions' beyond the traditional 401(k), such as employer premiums or IRA contributions, and its $36,918 is about $4,060 low. That gap matches understating taxable income by roughly $44k compared with the engine's $518,408."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used CA AGI of $534,270, which is $4,658 below the engine's $538,928.31, giving taxable income of about $515,000 instead of $518,408. It then rounded the tax down to $40,000 instead of applying the schedule, which gives $41,089 before the $114 exemption credit."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"Its stated method does not apply California's high-income itemized limitation (a $2,319.90 cutback in the engine). The engine's two personal exemption credits total $114 after the phaseout. Its answer is $163 below the engine's figure."
-us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Its federal AGI of $452,587 leaves out the $77,920 of qualified dividends and applies the full $10,911 net capital loss instead of the $3,000 limit. It then invented a $2,000 California loss limit and took the standard deduction instead of the $20,520 itemized deduction."
-us,scenario_005,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It took the about $11,300 standard deduction instead of itemizing $20,519.93 of property tax, charity and 2%-floor employee expenses after the cutback. It also used CA AGI of $534,270 instead of $538,928 and left out the $114 exemption credit."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave no computation. The correct path taxes $518,408 at CA MFJ rates up to 9.3%, for $41,089 less the $114 credit. Its $20,677 is about half that, an effective rate near 4% that no correct application of the CA schedule produces."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no figures, but it cites retirement-contribution deductions. Its $37,099 is about $3,880 low, which matches deducting roughly $42k more than the engine's $20,520 of limited itemized deductions from CA AGI of $538,928."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It excluded 'pretax health premiums' (the $13,369 of employer-sponsored insurance premiums) from wages, but the engine keeps them in federal and CA AGI. That, plus a lower starting AGI, left its $38,834 about $2,140 below the engine."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its taxable income of $519,427 is $1,019 above the engine's $518,408, so its limited itemized deduction or AGI is off by about $1k. That moved the result $49 above the engine's figure."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"Its $41,150 is above even the engine's pre-credit tax of $41,089.25. That is consistent with not subtracting the phased-down $114 exemption credit and with a small 'California additions' increase to income that the engine does not make."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It added tax-exempt interest as a California addition, but the engine's only CA adjustment to federal AGI is the $4,147.68 SALT-refund subtraction. Despite that addition it ended $809 low, so its wage base, deductions or exemption credits were also understated or overstated compared with $518,408 of taxable income and the $114 credit."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $520,901 subtracts the $13,369 of employer-sponsored insurance premiums, which the engine keeps in AGI ($538,928.31). It also used a $222 exemption credit instead of the phased-down $114."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It took the California standard deduction instead of itemizing $20,519.93, which overstated taxable income by about $9k. It also kept the state-tax refund in CA income, even though California subtracts the $4,147.68 refund."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It taxed $492,027 instead of $518,408, about $26k too much in deductions or too little CA AGI, and subtracted two full exemption credits instead of the phased-down $114."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $520,901 subtracts the $13,369 of employer-sponsored insurance premiums, which the engine keeps in AGI ($538,928.31). It also used a $222 exemption credit instead of $114."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It gave no figures. Its $40,200 is $775 low, which matches taxable income about $8k below the engine's $518,408 (for example, CA AGI of $534,270 instead of $538,928) plus a larger exemption credit than the phased-down $114."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no figures. Its $48,720 is about $7,745 high, which matches taxable income roughly $83k above the engine's $518,408. That is the effect of taxing the $70,000 tax-exempt pension and/or using the standard deduction instead of $20,520 of itemized deductions."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"It started from CA AGI of $534,270, which is $4,658 below the engine's $538,928.31, giving about $512k of taxable income instead of $518,408. It then subtracted a $280 exemption credit instead of the $114 left after the phaseout."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It deducted $22,934 of itemized deductions without California's high-income cutback. The engine reduces its $22,839.83 deduction total by $2,319.90 to $20,519.93. It also skipped the SALT-refund subtraction and used a $177 exemption credit instead of $114. Its taxable income was $515,484 against the engine's $518,408."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"It used CA AGI of $534,270 instead of $538,928.31 and left out the $2,986 of employee business expenses above the 2% floor before the cutback. It also subtracted $314 of exemption credits when the phaseout leaves only $114."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $525,049 equals $538,418 minus the $13,369 of employer-sponsored insurance premiums, which the engine does not subtract. It also itemized $23,201 without the high-income cutback and left out the $114 exemption credit."
-us,scenario_005,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It added the $70,000 tax-exempt private pension and the $8,961 of tax-exempt interest to CA AGI, inflating it to $604,010. The engine's CA AGI is federal AGI minus the SALT refund, $538,928.31. That pushed taxable income to $582,388 instead of $518,408."
-us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $40,975.25 reference."
-us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used CA AGI of $534,270, which is $4,658 below the engine's $538,928.31, giving taxable income of $512,433 instead of $518,408. It also subtracted a $258 exemption credit instead of the phased-down $114."
-us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It applied the federal $10,000 SALT cap, which California does not use, and deducted mortgage interest that was never listed. It then claimed deductions wipe out the tax, but over $518k of CA taxable income remains and is taxed at up to 9.3%."
-us,scenario_005,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It used CA AGI of $534,270 instead of $538,928.31 and itemized only $19,854 of property tax and charity. That leaves out the $2,986 of 2%-floor employee expenses and the $2,319.90 high-income cutback, putting taxable income at about $514,000 instead of $518,408."
-us,scenario_005,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It gave no figures. Its $39,300 is about $1,675 low, which matches taxable income roughly $18k below the engine's $518,408 (CA AGI of $538,928 minus $20,520 of limited itemized deductions) or exemption credits larger than $114."
-us,scenario_005,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It cut itemized deductions to about $11,912, a roughly $7,900 limitation instead of the engine's $2,319.90 cutback. It also left out the $2,986 of 2%-floor employee expenses and the SALT-refund subtraction, which pushed taxable income to $526,500 instead of $518,408."
-us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It left the $46,308 of traditional 401(k) deferrals in wages, kept the SALT refund that California subtracts, applied the full $10,911 capital loss, and took the standard deduction. It then got only $33,544 of tax on $565,375 when the 9.3% bracket gives about $45,500, and it subtracted an $892 exemption credit instead of $114."
-us,scenario_005,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It took the standard deduction instead of $20,520 of itemized deductions. It then reported $29,423 of tax on $495,603 of taxable income, when the CA MFJ schedule with 9.3% above about $145k gives roughly $39,000, so its bracket arithmetic is badly understated."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It imputed about $22,800 of mortgage interest from the $2.85M balance, but no interest paid was listed, so that input is $0. It then jumped from its own ~$38,832 bracket arithmetic to $45,856 with no supporting computation. The correct path itemizes only real estate tax, charity and the 2%-floor employee expenses ($20,520 after limitation), giving $518,408 taxable."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its CA AGI of ~$532,106 is about $6,800 below the correct $538,928 (federal AGI $543,076 less the $4,148 refund), so taxable income came out at $510,546 instead of $518,408. It also kept ~$210 of exemption credit, where the phaseout leaves only $114."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It itemized only the $17,280 real estate tax, cut to $15,907. It omitted the $2,574 charitable deduction and the $2,986 of employee business expenses above the 2% floor, both of which CA allows. Taxable income rose to $525,005 instead of $518,408, overstating the tax."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted an invented ~$23,000 deduction for federal income tax paid plus employer health premiums, neither of which CA allows. It applied a nonexistent 12% bracket above $325,000 and then took ~$21,621 of made-up nonrefundable credits. The only nonrefundable credit is the $114 phased-down exemption credit against $41,089 of tax."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It did not compute California tax and defaulted to $0. The household actually has $518,408 of CA taxable income, which produces $41,089 of tax before the $114 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It imputed ~$60,000 of mortgage interest from the $2.85M balance, but no interest was listed, so it counts as $0. That inflated itemized deductions to ~$79,854 and cut taxable income to $453,116 instead of $518,408. It also took a full $560 exemption credit instead of the phased-down $114."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It counted the $70,000 tax-exempt private pension as CA-taxable and imputed ~$35,000 of mortgage interest that was never listed. Its final $36,767 does not follow from its own ~$595,000 taxable income. The correct path excludes the pension, allows no mortgage interest, and taxes $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It partially included the tax-exempt private pension in CA income and arrived at ~$560,000 of taxable income, about $41,600 above the correct $518,408. Its $47,500 is a round estimate with no bracket computation behind it."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Its CA AGI of ~$532,106 is about $6,800 below the correct $538,928. It also applied too little of the high-income limitation, keeping $22,454 of itemized deductions instead of $20,520. Taxable income came to $509,652 instead of $518,408, and it kept $256 of exemption credit where only $114 survives the phaseout."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $534,270 is $4,658 below the correct $538,928. It skipped the 6%-of-excess-AGI itemized limitation, using $23,017 instead of $20,520. It also subtracted an unreduced ~$500 exemption credit instead of the phased-down $114, ending at $511,253 taxable instead of $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It imputed $50,000 of mortgage interest on the first $1M of the balance even though no interest was listed, and it omitted the 2%-floor employee expense deduction. Taxable income fell to $464,416 instead of $518,408. Its bracket arithmetic then gave $32,000, below what its own schedule produces on $464,416."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It itemized only property tax and charity (~$19.9k), omitting the $2,986 of employee expenses above the 2% floor. It started from a CA AGI of ~$534k instead of $538,928, giving ~$514k taxable instead of $518,408. It also zeroed out the exemption credit, which actually keeps $114 after the phaseout."
+us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Its taxable income was essentially right ($518,564 vs $518,408). However, its 2026 bracket thresholds produced ~$42,151 of tax, about $1,060 above the correct $41,089. It also claimed no nonrefundable credits, dropping the $114 phased-down exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its CA taxable income of $497,700 is about $20,700 below the correct $518,408 because AGI was understated relative to $538,928. It also subtracted a $300 exemption credit instead of the phased-down $114."
+us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its post-limitation taxable income of $512,889 is about $5,500 below the correct $518,408. It also said no California nonrefundable credits apply, dropping the $114 exemption credit that remains after the phaseout."
+us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $534,270 is $4,658 below the correct $538,928. It also left out the $2,986 of 2%-floor employee expenses, limiting itemized deductions to $18,663 and landing at $515,607 taxable. It then subtracted a $310 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Its $73,017 of itemized deductions includes ~$50,000 of imputed mortgage interest, even though no interest paid was listed. That dropped taxable income to $461,253 instead of $518,408. It also skipped the high-income limitation and took a $288 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $31,215 is about $9,760 below the correct $40,975, which at the 9.3% marginal rate equals understating CA taxable income by roughly $105,000 against the correct $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its CA taxable income of ~$501,848 is about $16,560 below the correct $518,408. The cause is an understated CA AGI relative to $538,928 combined with itemized deductions that skip the 6% high-income limitation."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $520,901 comes from subtracting the $13,369 of employer-sponsored insurance premiums from wages, a step the correct computation does not take (CA AGI is $538,928). It also left the $2,986 of 2%-floor employee expenses out of itemized deductions."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $33,816 is $7,160 below the correct $40,975, which at the 9.3% marginal rate equals understating CA taxable income by about $77,000 against the correct $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It itemized without the $2,320 high-income limitation and subtracted no exemption credit. Its bracket application on ~$518k also gave ~$42,150 of tax, $1,061 above the correct $41,089, so it overshot by $1,175 after the missing $114 credit."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no line items. Its $36,918 is $4,057 below the correct $40,975, which at the 9.3% marginal rate equals understating CA taxable income by about $43,600 against $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $534,270 is $4,658 below the correct $538,928, which put taxable income at ~$515,000 instead of $518,408. It then rounded the tax to $40,000 instead of computing $41,089 and subtracting the $114 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"It subtracted two personal exemption credits without the high-AGI phaseout, which reduces them to only $114 combined. It also itemized without the $2,320 high-income limitation, ending $163 below $40,975."
+us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left the $77,920 of qualified dividends out of AGI and deducted the full -$10,911 net capital loss instead of the $3,000 limit, then applied a made-up $2,000 CA loss cap. It also used the ~$11,000 standard deduction even though itemized deductions ($20,520) are larger, giving $438,439 taxable instead of $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used the ~$11,300 CA standard deduction instead of itemizing $20,520 (real estate tax, charity and 2%-floor employee expenses after the limitation), giving $522,970 taxable instead of $518,408. It also claimed no nonrefundable credits, dropping the $114 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $20,677 is about half of the 9.3%-bracket tax ($41,089 before the $114 credit) on $518,408 of CA taxable income."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $37,099 is $3,876 below the correct $40,975, which at the 9.3% marginal rate equals understating CA taxable income by about $42,000 against $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It took pretax health premiums off wages, removing the $13,369 of employer-sponsored insurance premiums, which the correct computation leaves in. This pulled CA AGI below $538,928 and produced $38,834 instead of $40,975."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its taxable income of ~$519,427 is about $1,000 above the correct $518,408 because it counted too little in itemized deductions after the 6% limitation. That left its result $49 above $40,975."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It gave no line items. Its $41,150 is $175 above the correct figure, which equals overstating taxable income by about $1,900 beyond $518,408 relative to the $20,520 of limited itemized deductions and the $114 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It added the $8,961 of tax-exempt interest to CA income, which the correct computation excludes. Its net $40,166 still lands $809 below $40,975, which equals taxable income about $8,700 short of the correct $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $520,901 subtracts the $13,369 of employer-sponsored insurance premiums from wages, whereas the correct CA AGI is $538,928. It also kept the prior-year bracket schedule instead of indexing to 2026 and took a $222 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used the CA standard deduction instead of the larger $20,520 of limited itemized deductions. It also kept the $4,148 state tax refund in CA income even though CA subtracts it, overstating taxable income above $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It overstated deductions and understated AGI, so taxable income came to ~$492,027, about $26,400 below the correct $518,408 (CA AGI $538,928 less $20,520 of limited itemized deductions)."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $520,901 subtracts the $13,369 of employer-sponsored insurance premiums from wages, whereas the correct CA AGI is $538,928. It also kept the latest prior-year bracket schedule unindexed for 2026 and took a $222 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It gave no line items. Its $40,200 is $775 below the correct figure, which equals understating CA taxable income by about $8,300 relative to the correct $518,408 (CA AGI $538,928 less $20,520 of limited itemized deductions)."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $48,720 is $7,745 above the correct figure, which equals about $83,000 of extra taxable income. That matches bringing the $70,000 tax-exempt pension into CA income instead of taxing $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $534,270 is $4,658 below the correct $538,928, giving ~$512,000 taxable instead of $518,408. It also subtracted a ~$280 exemption credit instead of the phased-down $114."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It skipped the 6% high-income itemized limitation, keeping $22,934 of itemized deductions instead of $20,520, so taxable income was $515,484 instead of $518,408. It also took a $177 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It left out the $2,986 of employee business expenses above the 2% floor, which CA still allows, and started from a CA AGI of $534,270 instead of $538,928. It also took $314 of exemption credits where the phaseout leaves only $114."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $525,049 is $13,879 below the correct $538,928, and it skipped the 6% itemized limitation, giving $501,848 taxable instead of $518,408. It also omitted the $114 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It added the $70,000 tax-exempt private pension and the $8,961 of tax-exempt interest to CA AGI ($604,010), though both stay out of CA income. This inflated taxable income to $582,388 instead of $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against $40,975.25."
+us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Its CA AGI of $534,270 is $4,658 below the correct $538,928, leaving taxable income at $512,433 instead of $518,408. It also subtracted $258 of exemption credit where only $114 survives the phaseout."
+us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It imputed mortgage interest on the $2.85M balance even though none was listed, and it applied the federal $10,000 SALT cap, which California does not have. From that it concluded the tax was eliminated, but CA taxable income is $518,408, producing $41,089 of tax."
+us,scenario_005,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It itemized only property tax and charity ($19,854), omitting the $2,986 of 2%-floor employee expenses. It started from a CA AGI of $534,270 instead of $538,928 and rounded taxable income to $514,000 instead of $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It gave no line items. Its $39,300 is $1,675 below the correct figure, which equals understating CA taxable income by about $18,000 relative to the correct $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It applied the high-income itemized limitation far too heavily, cutting itemized deductions to $11,912. The actual 6%-of-excess reduction is only $2,320, leaving $20,520, and it also omitted the 2%-floor employee expenses, ending at $526,500 taxable instead of $518,408."
+us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It skipped the $46,308 of 401(k) deferrals, kept the full -$10,911 capital loss instead of the $3,000 limit, left the $4,148 refund in income and used the standard deduction. On top of that, its brackets gave only $33,544 of tax on $565,375, where the 9.3% bracket yields about $45,000. It also subtracted an $892 exemption credit instead of $114."
+us,scenario_005,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It used the CA standard deduction instead of itemizing $20,520. Its bracket arithmetic also gave only $29,423 on $495,603 of taxable income, where CA's schedule through the 9.3% bracket yields about $39,000; the correct base is $518,408 with $41,089 of tax."
us,scenario_005,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It treated the $19,285 of SSDI as fully non-taxable. Provisional income of $44,922.50 is above the $34,000 threshold, so $13,784 of it is taxable. It also used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100, so its taxable income came out at $20,680 instead of $32,964."
us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It assumed taxable SSDI was about $0 'given low other income.' Provisional income of $44,922.50 is above $34,000, so $13,784 is taxable. That left out $13,784 of AGI, and its $1,900 is not even the right 10%/12% tax on the $19,180 it was left with."
@@ -374,57 +379,58 @@ us,scenario_007,head_medicare_eligible,qwen3.8-max,prompt_ambiguity,age_disabili
us,scenario_007,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model first recognized that no wage was listed but then asserted $1,516.30 of employee Social Security and Medicare tax on unspecified “applicable taxable wages.” The listed Social Security disability and private pension payments are not employee FICA wages, and the 50 weekly hours cannot create wage income when the unlisted wage rate and annual wages are zero."
us,scenario_007,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract."
us,scenario_007,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reached the $19,180 base but taxed it at a flat 5.695% with no $4,811 zero-rate amount, then raised its own $1,092 to $1,571 with no basis. It also skipped the $2,080 Idaho health insurance premium subtraction and the $10 permanent building fund tax. Correct: (17,100 − 4,811) × 5.3% + $10 = $661.32."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The model subtracted Social Security and the $16,100 standard deduction but skipped the $2,080 Idaho health insurance premium subtraction, so it used $19,180 instead of $17,100. It also applied 5.3% to the whole amount with no $4,811 zero-rate amount and left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"The model invented a roughly $41,110 Idaho pension exclusion and used it to wipe out the $35,280 private pension. Idaho's retirement benefits deduction covers qualifying public pensions for filers aged 65+ (or 62+ and disabled), so a 56-year-old's private pension stays taxable. Idaho taxable income is $17,100 and the tax is $661.32."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model asserted zero liability without computing anything. After the Social Security subtraction, the $2,080 health premium subtraction and the $16,100 standard deduction, $17,100 of taxable income remains, which produces $651.32 at 5.3% above $4,811, plus the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly got to $19,180 after the Social Security subtraction and standard deduction, then used a flat 5.695% with no $4,811 zero-rate amount and raised its $1,092 to $1,379 with no basis. It skipped the $2,080 health insurance premium subtraction, the 5.3% 2026 rate and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used a $15,750 standard deduction instead of $16,100 and the outdated 5.695% rate above $4,673 instead of 5.3% above $4,811. It then raised its own $846 to $1,300 with no basis. It also skipped the $2,080 Idaho health insurance premium subtraction and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model skipped the $2,080 health insurance premium subtraction, so it used $19,180 instead of $17,100, and its own setup (5.3% above about $4,700) gives about $767, not the $1,112 it reported. It also left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model got the rate and zero bracket right (5.3% above $4,811) but skipped Idaho's $2,080 subtraction for health insurance premiums, so it taxed $19,180 instead of $17,100. It also left out the $10 permanent building fund tax, which is part of the $661.32."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model said Idaho has no Social Security subtraction and taxed 85% of the SSDI. Idaho fully subtracts Social Security included in federal AGI. It also used a personal exemption Idaho does not have, the outdated 5.695% rate with no $4,811 zero bracket, and took the refundable grocery credit as a nonrefundable offset. It skipped the $2,080 health premium subtraction."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction instead of $16,100 and a flat 5.695% instead of 5.3% above $4,811, then raised its own $1,178 to $1,800 with no basis. It skipped the $2,080 health insurance premium subtraction and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model taxed $19,180 at a flat 5.3%. It skipped the $2,080 health insurance premium subtraction (taxable income is $17,100), did not exempt the first $4,811 at a zero rate, and left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model never subtracted the $13,784 of taxable Social Security, though Idaho excludes it, and used a $14,000 standard deduction instead of $16,100. It taxed $35,064 at a flat 5.695% instead of 5.3% above $4,811, skipped the $2,080 health premium subtraction, and left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used the 2023 flat rate of 5.8% with no $4,811 zero bracket and a $15,300 standard deduction instead of 5.3% and $16,100. It also skipped the $2,080 health insurance premium subtraction and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used a $15,400 standard deduction instead of $16,100 and taxed all $19,880 at 5.695%. The 2026 schedule is 5.3% above $4,811. It also skipped the $2,080 health premium subtraction and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model used an $8,100 standard deduction instead of the $16,100 Idaho conforming amount and taxed $27,180 at a flat 5.3% with no $4,811 zero bracket. It also skipped the $2,080 health premium subtraction and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the outdated 5.8% flat rate with no $4,811 zero bracket and a $15,700 standard deduction instead of $16,100. It skipped the $2,080 health insurance premium subtraction and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model counted 'taxable disability income' in the Idaho base. Idaho fully subtracts the $13,784 of Social Security disability included in federal AGI. It also never applied the $2,080 health premium subtraction or the 5.3%-above-$4,811 schedule that produces $651.32 plus the $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model excluded Social Security, but its $1,246 matches a flat rate applied to roughly $21,000–$23,500 of pension income with no zero bracket. The correct path subtracts $2,080 of health premiums and the $16,100 standard deduction to reach $17,100, taxes it at 5.3% above $4,811, and adds the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model subtracted a smaller standard deduction plus a personal exemption Idaho does not have, reaching $21,830 instead of $17,100. It taxed that at a flat 5.695% instead of 5.3% above $4,811. It skipped the $2,080 health premium subtraction and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model's $3,371 is consistent with taxing gross income, including the SSDI, at a flat rate with almost no deductions. Idaho subtracts the taxable Social Security and the $2,080 of health premiums and allows the $16,100 standard deduction, leaving $17,100 taxed at 5.3% above $4,811, plus the $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's $1,115.08 equals 5.695% × $19,580, meaning a $15,700 standard deduction and the outdated flat rate with no zero bracket. It skipped the $2,080 health insurance premium subtraction, the 5.3% rate above $4,811 and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's explanation computes 5.695% × $19,180 = $1,092.30 but it submitted $1,894. Its method also uses the wrong rate and no $4,811 zero bracket, skips the $2,080 health premium subtraction, and leaves out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model taxed $19,930 at the outdated flat 5.8%, using a $15,350 standard deduction instead of $16,100 and no zero bracket. The 2026 schedule is 5.3% above $4,811. It skipped the $2,080 health premium subtraction and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"The model correctly applied the $2,080 health premium subtraction but used a $14,600 standard deduction instead of $16,100. It also used the 2024 schedule (5.695% above $4,673) instead of the 2026 schedule (5.3% above $4,811) and left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model applied Idaho's retirement benefits deduction to the $35,280 private pension. That deduction covers qualifying public pensions for filers aged 65+ or 62+ and disabled, so it does not apply to a 56-year-old's private pension. Idaho taxable income is $17,100, and the tax is $661.32 including the $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model correctly added the $10 permanent building fund tax but taxed all $19,530 at 5.3% with no $4,811 zero bracket. It used a $15,750 standard deduction instead of $16,100 and skipped the $2,080 health insurance premium subtraction that brings taxable income to $17,100."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model claimed the standard deduction fully offsets income. The $35,280 private pension stays in Idaho AGI after the Social Security subtraction, and after the $2,080 health premium subtraction and the $16,100 standard deduction, $17,100 of taxable income remains, producing $661.32."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model reported zero tax with no derivation. The taxable $35,280 pension leaves $17,100 of Idaho taxable income after the $2,080 health premium subtraction and the $16,100 standard deduction, which is taxed at 5.3% above $4,811, plus the $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model skipped the $2,080 Idaho health insurance premium subtraction, so it taxed $19,180 instead of $17,100. It used a $5,000 zero bracket instead of $4,811 and left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $1,885 equals 5.3% of about $35,570, which matches federal AGI less a standard deduction with no Idaho Social Security subtraction. Idaho removes the $13,784 of taxable Social Security and $2,080 of health premiums, leaving $17,100 taxed at 5.3% above $4,811, plus $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model taxed $19,180 at a flat 5.3%. It skipped the $2,080 health insurance premium subtraction (taxable income is $17,100), did not apply the $4,811 zero-rate amount, and left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $1,759.89 is consistent with applying 5.3% to about $33,206 of taxable income. Idaho removes the taxable Social Security and the $2,080 of health premiums, leaving $17,100, which is taxed only above $4,811, plus the $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model got the whole calculation right ($17,100 taxable, 5.3% above $4,811 = $651.32) but left out Idaho's $10 permanent building fund tax. That tax is part of state income tax before refundable credits and brings the total to $661.32."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model's taxable income of about $35,572 means it never removed the $13,784 of taxable Social Security from federal AGI. It also skipped the $2,080 health premium subtraction, applied 5.3% with no $4,811 zero bracket, and left out the $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model skipped the $2,080 Idaho health insurance premium subtraction, so it taxed $19,180 instead of $17,100. It used a $4,990 zero-rate amount instead of $4,811 and left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model used the correct 5.3% rate and $4,811 zero bracket but skipped the $2,080 Idaho health insurance premium subtraction, so it taxed $19,180 instead of $17,100. It also left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"The model applied 5.3% to the whole $19,180. It skipped the $2,080 health premium subtraction (taxable income is $17,100), did not apply the $4,811 zero-rate amount, and left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's $1,477 matches a flat rate applied to pension income less a deduction far smaller than the $16,100 Idaho conforming standard deduction. It also skipped the $2,080 health premium subtraction, the $4,811 zero bracket and the $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an $8,535 projected standard deduction instead of the $16,100 Idaho conforming amount. It taxed $26,745 at a flat 5.695% instead of 5.3% above $4,811, skipped the $2,080 health premium subtraction, and left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used a standard deduction of about $8,477 instead of $16,100 and taxed $26,803 at a flat 5.695% instead of 5.3% above $4,811. It skipped the $2,080 health premium subtraction and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model used the TCJA-sunset $8,550 standard deduction, but the TCJA standard deduction was made permanent, so the 2026 single amount Idaho conforms to is $16,100. It also taxed all income at 5.3% with no $4,811 zero bracket, skipped the $2,080 health premium subtraction, and left out the $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model's $36,252 base keeps the federally taxable Social Security in Idaho income, but Idaho subtracts it in full. It taxed that base at an outdated flat 5.8% instead of 5.3% above $4,811 and skipped the $2,080 health premium subtraction and the $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model taxed the $32,964 federal taxable income base without Idaho's subtraction of the $13,784 of taxable Social Security or the $2,080 of health premiums. It applied 5.3% with no $4,811 zero bracket and left out the $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for this output, so there was nothing to score against the $661.32."
-us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model got to $19,180 by using a $13,600 deduction/exemption and a $2,500 zero bracket instead of the $16,100 standard deduction and the $4,811 zero-rate amount. It skipped the $2,080 health premium subtraction and the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model claimed Idaho has no individual income tax. Idaho levies a 5.3% income tax above a $4,811 zero-rate amount plus a $10 permanent building fund tax, which gives $661.32 on $17,100 of taxable income."
-us,scenario_007,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"The model's $34,814 base matches federal AGI less a deduction of about $14,250 with no Idaho Social Security subtraction. Idaho removes the $13,784 of taxable Social Security and $2,080 of health premiums, leaving $17,100 taxed at 5.3% above $4,811, plus $10 PBF."
-us,scenario_007,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"The model applied a flat 5.3% to $19,180. It skipped the $2,080 health insurance premium subtraction (taxable income is $17,100), did not apply the $4,811 zero-rate amount, and left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model got to $19,180 and applied a flat 5.3%. It skipped Idaho's $2,080 health insurance premium subtraction, did not apply the $4,811 zero-rate amount, and left out the $10 permanent building fund tax."
-us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $19,285 SSDI as taxable and never applied Idaho's Social Security subtraction. It used a $14,600 deduction and an outdated 5.8% rate, and subtracted the grocery credit, which is refundable and does not belong in this output. It then cut its own $2,198 to $1,707 with no basis."
-us,scenario_007,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used a graduated schedule that matches no Idaho rate schedule and took an extra $5,200 deduction on top of the federal standard deduction. Idaho now has a single 5.3% rate above $4,811 on $17,100 of taxable income, after the $2,080 health premium subtraction, plus the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reached $19,180 of taxable income but applied the old 5.695% rate with no zero-rate bracket, instead of 5.3% above about $4,811. It then raised the result to $1,571 with no arithmetic behind it. It also missed the $2,080 health insurance premium subtraction and the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model applied 5.3% to all $19,180 and skipped Idaho's zero-rate band of about $4,811. It also did not subtract the $2,080 of health insurance premiums (correct taxable income is $17,100) and left out the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"The rate, the $4,811 zero bracket and the $10 permanent building fund tax were all right. The model missed Idaho's subtraction for the $2,080 of health insurance premiums paid, so it taxed $19,180 instead of $17,100 and came out $110.24 too high."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented a retirement-income exclusion that wiped out the $35,280 private pension. Idaho's retirement benefits deduction covers only certain public and military pensions, and only at age 65 (or 62 if disabled), so it does not apply to a private pension at age 56. The real result is $17,100 of taxable income and $661.32 of tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model asserted zero liability without any calculation. The $35,280 private pension is still taxable in Idaho: after the $2,080 premium subtraction and the $16,100 standard deduction, $17,100 is taxed at 5.3% above about $4,811, plus the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly got $19,180 after subtracting Social Security, then used the outdated 5.695% rate with no zero bracket and pushed the result up to $1,379 with no basis. It also missed the $2,080 health premium subtraction, the 5.3% rate above about $4,811, and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used a $15,750 standard deduction instead of $16,100 and the old 5.695% rate instead of 5.3%. It skipped the $2,080 health premium subtraction, and its own figure of $846 was then raised to $1,300 for no stated reason. It also left out the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model described 5.3% above about $4,700 on $19,180, which comes to roughly $767, yet it reported $1,112. It also missed the $2,080 health premium subtraction (taxable income should be $17,100) and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The rate and the $4,811 zero bracket were right. The model missed Idaho's subtraction for the $2,080 of health insurance premiums paid (taxable income is $17,100, not $19,180) and left out the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model taxed 85% of the disability benefit in Idaho, but Idaho subtracts all federally taxable Social Security. It also invented a $4,950 personal exemption, used the 5.695% rate, and took the refundable grocery credit as nonrefundable. The premium subtraction, the zero bracket and the $10 building fund tax were all missed."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction instead of $16,100 and the 5.695% rate instead of 5.3% above about $4,811. It then raised its own $1,178 to $1,800 with no basis. It also missed the $2,080 premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model applied 5.3% to all $19,180 and skipped the zero-rate band of about $4,811. It did not subtract the $2,080 of health premiums (taxable income is $17,100) and left out the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model never applied Idaho's subtraction of the $13,784 of taxable Social Security, so that amount was taxed. It also used a $14,000 standard deduction instead of $16,100 and the 5.695% rate instead of 5.3% above about $4,811. The premium subtraction and the $10 building fund tax were missed as well."
+us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied an outdated flat 5.8% rate with no zero bracket and used a $15,300 standard deduction instead of $16,100. It also missed the $2,080 health premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used a $15,400 standard deduction instead of $16,100 and taxed everything at 5.695% instead of 5.3% above about $4,811. It also missed the $2,080 premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model used an $8,100 standard deduction, but Idaho conforms to the federal $16,100 single deduction for 2026. It also skipped the zero-rate band of about $4,811, the $2,080 premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied an outdated flat 5.8% rate with no zero bracket and a $15,700 standard deduction instead of $16,100. It also missed the $2,080 health premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the disability Social Security as taxable Idaho income, but Idaho subtracts all federally taxable Social Security. The correct path also subtracts the $2,080 of premiums and the $16,100 standard deduction, leaving $17,100 taxed at 5.3% above about $4,811, plus $10."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model did exclude Social Security, but $1,246 matches roughly $21,900 taxed at the old 5.695% rate with no zero bracket. The correct calculation taxes $17,100 (after the $2,080 premium subtraction and the $16,100 deduction) at 5.3% above about $4,811, plus the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted a standard deduction plus a personal exemption that totals only $13,450. Idaho has no personal exemption, and the conforming 2026 deduction is $16,100. The model also used 5.695% instead of 5.3% above about $4,811, and missed the $2,080 premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The $3,371 answer has no calculation behind it and implies about $60,000 of taxable income. It is consistent with taxing Social Security and taking no deductions. The correct base is $17,100 after the Social Security subtraction, the $2,080 premium subtraction and the $16,100 deduction, taxed at 5.3% above about $4,811, plus $10."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The $1,115.08 answer equals $19,580 (a $15,700 deduction instead of $16,100) times the outdated flat 5.695% rate, with no zero bracket. The model also missed the $2,080 premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's explanation applies the outdated 5.695% rate to all $19,180 to get $1,092.30, yet it submitted $1,894. It missed the 5.3% rate above about $4,811, the $2,080 premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model applied an outdated flat 5.8% rate with no zero bracket and a $15,350 standard deduction instead of $16,100. It also missed the $2,080 health premium subtraction and the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"The model correctly took the $2,080 premium subtraction but used the 2024 $14,600 standard deduction instead of $16,100. It applied 5.695% above $4,673 instead of 5.3% above about $4,811, and left out the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model applied Idaho's retirement benefits deduction to the $35,280 private pension. That deduction covers only certain public and military pensions, and the claimant must be 65 (or 62 if disabled), so it does not apply at age 56. The real result is $17,100 of taxable income and $661.32 of tax."
+us,scenario_007,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model correctly added the $10 permanent building fund tax but applied 5.3% to all of its taxable income with no zero-rate band of about $4,811. It also used a $15,750 deduction instead of $16,100 and missed the $2,080 health premium subtraction."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model claimed the standard deduction wipes out taxable income, but it does not. After subtracting Social Security and the $2,080 of premiums, Idaho AGI is $33,200, and the $16,100 deduction still leaves $17,100 taxable at 5.3% above about $4,811, plus $10."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model asserted zero tax without calculating it, but the $35,280 private pension is fully taxable in Idaho. After the $2,080 premium subtraction and the $16,100 deduction, $17,100 is taxed at 5.3% above about $4,811, plus the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model missed Idaho's subtraction for the $2,080 of health insurance premiums paid and used a $5,000 zero bracket instead of about $4,811. It also left out the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The $1,885 answer equals 5.3% of about $35,570, which means the model did not properly subtract Social Security or did not apply the deductions. The correct base is $17,100 after the full Social Security subtraction, the $2,080 premium subtraction and the $16,100 deduction, taxed above about $4,811, plus $10."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model taxed all $19,180 at 5.3% and skipped the zero-rate band of about $4,811. It also missed the $2,080 health premium subtraction (taxable income is $17,100) and the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The $1,759.89 answer equals 5.3% of $33,205.53, which is federal taxable income including taxable Social Security. The model never applied Idaho's full Social Security subtraction. It also missed the $2,080 premium subtraction, the zero-rate band of about $4,811 and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model computed the $651.32 income tax correctly but left out Idaho's $10 permanent building fund tax. That tax is part of state income tax before refundable credits, so the total is $661.32."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model's $35,572 of taxable income is federal AGI with 85% of the disability benefit included, less $16,100. It never applied Idaho's full Social Security subtraction or the $2,080 premium subtraction. It also skipped the zero-rate band of about $4,811 and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model missed Idaho's subtraction for the $2,080 of health insurance premiums paid and used a zero bracket of about $4,990 instead of about $4,811. It also left out the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The rate and the $4,811 zero band were right. The model missed Idaho's subtraction for the $2,080 of health premiums paid (taxable income is $17,100, not $19,180) and left out the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"The model taxed all $19,180 at 5.3% and skipped the zero-rate band of about $4,811. It also missed the $2,080 premium subtraction and the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $1,477 answer implies roughly $26,000 to $28,000 of taxable income, which is consistent with a pre-2018 sized standard deduction instead of the conforming $16,100. The model also missed the $2,080 premium subtraction, the 5.3% rate above about $4,811 and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an $8,535 standard deduction, but Idaho conforms to the federal 2026 single deduction of $16,100. It also used 5.695% instead of 5.3% above about $4,811, and missed the $2,080 premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model applied a standard deduction of only about $8,477 instead of the conforming $16,100, giving $26,803 of taxable income. It also used 5.695% instead of 5.3% above about $4,811, and missed the $2,080 premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model assumed the TCJA standard deduction expired, giving $8,550. The OBBBA made the higher deduction permanent ($16,100 for single filers in 2026), and Idaho conforms. The model also skipped the zero-rate band of about $4,811, the $2,080 premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model's $36,252 of taxable income keeps 85% of the Social Security benefit, so it never applied Idaho's full Social Security subtraction. It also used a $15,420 deduction and an outdated 5.8% rate with no zero bracket, and missed the $2,080 premium subtraction and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model taxed federal taxable income of $32,964, which still includes the $13,784 of taxable Social Security that Idaho fully subtracts. It also missed the $2,080 premium subtraction, the zero-rate band of about $4,811 and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used a $13,600 deduction and a $2,500 zero bracket instead of the $16,100 standard deduction and a zero band of about $4,811. It also missed Idaho's $2,080 health premium subtraction and the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model claimed Idaho has no individual income tax. Idaho does: a flat 5.3% rate above a zero-rate amount, plus a $10 permanent building fund tax, which gives $661.32 on $17,100 of taxable income."
+us,scenario_007,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"The model's $34,814 of taxable income keeps most of the taxable Social Security, so it never applied Idaho's full subtraction of the $13,784. It also missed the $2,080 premium subtraction, the zero-rate band of about $4,811 and the $10 building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model taxed all $19,180 at 5.3% and skipped the zero-rate band of about $4,811. It also missed the $2,080 health premium subtraction (taxable income is $17,100) and the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model applied 5.3% to all $19,180 and skipped Idaho's zero-rate band of about $4,811. It also missed the $2,080 health insurance premium subtraction and the $10 permanent building fund tax."
+us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model put 100% of the disability benefit into AGI and never applied Idaho's full Social Security subtraction. It used an outdated 5.8% rate and treated the refundable grocery credit as nonrefundable, then landed on $1,707.48 with no arithmetic to support it. The premium subtraction, the zero bracket and the $10 building fund tax were all missed."
+us,scenario_007,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model invented a graduated rate schedule and took an extra $5,200 Idaho deduction on top of federal taxable income. Idaho uses one conforming $16,100 standard deduction and a flat 5.3% rate above about $4,811. The model also missed the $2,080 premium subtraction and the $10 building fund tax."
us,scenario_007,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly picked the Idaho Grocery Credit as the refundable credit for a single filer under 65. However, it used the old $120 per-person amount instead of the $155 that applies from 2025."
us,scenario_007,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model said Idaho has no refundable individual income tax credits. That overlooks the Idaho Grocery Credit, which is refundable and worth $155 for this resident under 65 in 2026."
us,scenario_007,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,The model said no refundable Idaho credits apply and left out the refundable Idaho Grocery Credit. This full-year resident under 65 who does not receive SNAP qualifies for $155 in 2026.
@@ -531,11 +537,12 @@ us,scenario_008,child5_chip_eligible,ox-alpha,llm_error,health_coverage,False,"T
us,scenario_008,child5_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model computed MAGI at about 59% FPL for a household of eight, checked it against NJ CHIP's 355% ceiling, and then cleared the child because ""no disqualifying health coverage is listed."" The disqualifier is computed, not listed: at 59% FPL a 5-year-old is Medicaid-eligible under the under-6 (YOUNG_CHILD) category, and Medicaid eligibility — not enrollment — bars CHIP."
us,scenario_008,child5_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"The model ran a single test — MAGI of about $30,800 is ""within NJ CHIP income limits"" — and answered yes. Being within CHIP limits is not sufficient: the child must first fail Medicaid, and at this income a 5-year-old clears New Jersey's Medicaid threshold for the under-6 band, so Medicaid eligibility under the YOUNG_CHILD category makes the child CHIP-ineligible."
us,scenario_008,child5_early_head_start_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated Early Head Start as serving children from birth through age 5. Because Child 5 is age 5, the child is outside Early Head Start's under-age-3 category; income and housing assistance do not cure that categorical age failure."
-us,scenario_008,child5_head_start_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"It limited preschool Head Start to ages 3-4 and ruled out Child 5 on the grounds that 5-year-olds are of kindergarten age. Head Start eligibility actually runs from age 3 until the age of compulsory school attendance (6 in NJ), and PolicyEngine includes age 5. Because it stopped at the wrong age cutoff, it never applied the SNAP categorical pathway, which the household's SNAP receipt satisfies."
-us,scenario_008,child5_head_start_eligible,claude-sonnet-5.5,llm_error,age_disability,False,"It set the Head Start preschool age range at 3-4 and treated age 5 as 'at or above' it. The eligible range actually extends to the compulsory school age of 6, so 5-year-olds qualify. It therefore never applied the SNAP categorical eligibility or the income test, both of which Child 5 meets."
-us,scenario_008,child5_head_start_eligible,deepseek-v4-pro,llm_error,age_disability,False,"It declared age 5 'past the usual Head Start preschool age range'. Under the Head Start rule that PolicyEngine encodes, children are eligible from age 3 until the age of compulsory school attendance, so a 5-year-old is inside the range. Because it excluded Child 5 on age, it skipped the SNAP categorical eligibility that makes the child eligible."
-us,scenario_008,child5_head_start_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"It assumed that turning 5 ages a child out of preschool Head Start. The program actually covers children from age 3 until the age of compulsory school attendance (6 in NJ), and PolicyEngine counts age 5 as eligible. Its answer ignores the household's SNAP-based categorical eligibility, which settles the income side."
-us,scenario_008,child5_head_start_eligible,qwen3.8-max,llm_error,age_disability,False,"It placed age 5 outside the Head Start preschool age range. The rule actually runs from age 3 until the age of compulsory school attendance, so a 5-year-old is age-eligible. It never evaluated the SNAP categorical pathway or the income test, both of which qualify Child 5."
+us,scenario_008,child5_head_start_eligible,claude-fable-5.2,llm_error,age_disability,False,"Treated Head Start preschool eligibility as ages 3–4 only and excluded the 5-year-old, even though the program serves children from age 3 until compulsory school age, which includes age 5. Because of that, it never applied the SNAP categorical pathway, even while noting the household's low income, so it answered 0 instead of 1."
+us,scenario_008,child5_head_start_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"Assumed that 5-year-olds are kindergarten age and outside Head Start. In fact, the rule covers children from age 3 up to the age required to attend school, and PolicyEngine's age range includes age 5. It skipped the SNAP categorical pathway (the household receives about $1,260/month in SNAP) and the income test, which would have made Child 5 eligible."
+us,scenario_008,child5_head_start_eligible,claude-sonnet-5.5,llm_error,age_disability,False,"Applied a 3–4 age range and treated age 5 as 'at or above' the cutoff. The Head Start age band runs through age 5 (up to compulsory school age), so Child 5 passes the age test. The child is also categorically eligible through the household's SNAP receipt, so the answer is 1."
+us,scenario_008,child5_head_start_eligible,deepseek-v4-pro,llm_error,age_disability,False,"Called age 5 'past the usual Head Start preschool age range.' The age criterion actually runs from 3 until school-attendance age, so it includes 5-year-olds. It never applied the SNAP categorical pathway or the income test, both of which Child 5 meets."
+us,scenario_008,child5_head_start_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"Assumed that turning 5 ages a child out of preschool Head Start. Eligibility actually continues until the age required to attend school, so a 5-year-old remains eligible. It missed that the household's SNAP receipt makes Child 5 categorically eligible and that household income (about $30.6K for 8 people) also passes the income test."
+us,scenario_008,child5_head_start_eligible,qwen3.8-max,llm_error,age_disability,False,"Placed age 5 outside the Head Start preschool range by using a 3–4 cap. The program's age band covers children from 3 until compulsory school age, which includes 5. With SNAP receipt providing categorical eligibility and income under the poverty-guideline test, Child 5 is eligible (value 1)."
us,scenario_008,child5_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separate explicit Medicaid-triggering fact instead of applying the age-based YOUNG_CHILD pathway. Child 5's age of 5 and MAGI of 0.55 times FPL satisfy that pathway, yielding eligibility."
us,scenario_008,child5_wic_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model correctly said 'children under 5 years old are eligible' but then counted a 5-year-old as under 5. WIC child eligibility ends at the 5th birthday, so Child 5 is categorically ineligible. The income and housing-assistance reasoning has no bearing on that."
us,scenario_008,child5_wic_eligible,claude-haiku-5.5,llm_error,age_disability,False,"The model called a 5-year-old 'a child under 5', which contradicts the stated age. WIC's child category ends at the 5th birthday, so Child 5 is not eligible no matter how much the household earns."
@@ -621,110 +628,112 @@ us,scenario_008,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibilit
us,scenario_008,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly found the head in the expansion category with income below 138% FPL. It then applied a made-up rule that receiving housing assistance disqualifies adults from Medicaid. No such exclusion exists, so the head's ACA adult expansion eligibility at 0.55× FPL stands."
us,scenario_008,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model explicitly transferred the infant's categorical eligibility to the Head through household membership. WIC requires the applicant personally to be a pregnant, postpartum, or breastfeeding woman, infant, or child under age 5; the 42-year-old Head meets none of those categories and is therefore ineligible despite household income below the limit."
us,scenario_008,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_008,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It correctly included NJ UI ($102.51) and workforce ($11.39) but set the employee TDI rate to 0% instead of 0.19% ($50.92). It also used a 0.09% FLI rate ($24.12) instead of 0.23% ($61.64). Together these understate the NJ contributions by $88.44. Its listed components sum to $138.02, but it reported them as $128.65, which brings the shortfall to $97.81."
-us,scenario_008,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It counted only federal FICA ($2,050.20). It omitted New Jersey's mandatory employee TDI, FLI, UI and workforce contributions, which total $226.46 on the spouse's $26,800 wages."
-us,scenario_008,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"After a confused attempt at the NJ UI rate, it added an invented $2.12 'minimum assessment' to FICA. It should have added NJ TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce 0.0425%, totaling $226.46."
-us,scenario_008,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"It stopped at federal FICA of $2,050.20. It never added New Jersey's mandatory employee TDI/FLI/UI/workforce contributions, which total $226.46."
-us,scenario_008,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It named NJ UI, SDI and FLI but wrongly took the 2026 SDI/TDI employee rate as 0%. It then dropped all NJ contributions because of rate uncertainty, missing $226.46 (TDI 0.19%, FLI 0.23%, UI 0.3825%, WF 0.0425%)."
-us,scenario_008,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It assumed NJ employee UI/SDI/FLI contributions were not modeled and set them to $0. Those mandatory employee state payroll taxes add $226.46 on $26,800 of wages."
-us,scenario_008,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It explicitly excluded NJ employee UI/TDI/FLI from the measure and rounded FICA to $2,050. The output includes $226.46 of NJ employee TDI, FLI, UI and workforce contributions."
-us,scenario_008,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It applied only 7.65% federal FICA. It omitted the $226.46 of mandatory NJ employee contributions (TDI $50.92, FLI $61.64, UI $102.51, WF $11.39)."
-us,scenario_008,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It wrongly asserted that NJ TDI is employer-funded with a 0% worker rate and excluded all NJ employee contributions. NJ withholds worker TDI (0.19%), FLI (0.23%), UI (0.3825%) and workforce (0.0425%), which total $226.46."
-us,scenario_008,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It estimated NJ employee UI/DI/FLI at about $201 but then set them to 0, citing cap uncertainty. Those contributions are $226.46 because the $26,800 wages sit below every NJ wage base."
-us,scenario_008,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It assumed NJ employee payroll taxes were not modeled. It therefore omitted $226.46 of TDI, FLI, UI and workforce contributions on the spouse's wages."
-us,scenario_008,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It computed only federal FICA ($2,050.20, submitted as $2,050). It omitted New Jersey's $226.46 of mandatory employee TDI/FLI/UI/workforce contributions."
-us,scenario_008,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It used outdated NJ rates of 0.10% TDI and 0.09% FLI instead of the 2026 rates of 0.19% and 0.23%. It also omitted the employee UI (0.3825%, $102.51) and workforce (0.0425%, $11.39) contributions entirely."
-us,scenario_008,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It put NJ employee contributions at $154.10 instead of $226.46. If UI plus workforce account for $113.90, that leaves only $40.20 for TDI and FLI, far below the 2026 worker rates of 0.19% TDI ($50.92) and 0.23% FLI ($61.64)."
-us,scenario_008,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It counted only Social Security and Medicare ($2,050.20). It omitted New Jersey's mandatory employee TDI, FLI, UI and workforce contributions, which total $226.46."
-us,scenario_008,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"Its 0.515% NJ rate covers UI/workforce (0.425%) plus a 0.09% FLI rate. It omitted the 0.19% worker TDI ($50.92) and understated FLI, which is 0.23% ($61.64) in 2026."
-us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It applied only the 7.65% federal FICA rate. It omitted the $226.46 of mandatory NJ employee TDI, FLI, UI and workforce contributions."
-us,scenario_008,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It estimated NJ employee contributions at about $138, which matches UI/workforce plus a 0.09% FLI rate. That omits the 0.19% TDI and understates FLI at its 2026 rate of 0.23%; the correct NJ total is $226.46."
-us,scenario_008,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It included only NJ UI and FLI, totaling $123.80. It omitted worker TDI at 0.19% ($50.92) and did not reach the 2026 FLI rate of 0.23% plus the 0.0425% workforce contribution, so it fell $102.66 short of the $226.46 NJ total."
-us,scenario_008,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,It applied only 7.65% FICA to the spouse's wages. It omitted New Jersey's $226.46 of mandatory employee TDI/FLI/UI/workforce contributions.
-us,scenario_008,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"Its NJ SUI/FLI figure of $138.02 is 0.515% of wages: UI/workforce at 0.425% plus FLI at 0.09%. It omitted worker TDI (0.19%, $50.92) and understated FLI, which is 0.23% in 2026."
-us,scenario_008,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"It counted only NJ FLI, UI and workforce contributions, $139.32 in total. It left out worker TDI at 0.19% ($50.92) and used an FLI rate far below the 2026 rate of 0.23%."
-us,scenario_008,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It counted only Social Security and Medicare. It omitted New Jersey's mandatory employee TDI, FLI, UI and workforce contributions, which total $226.46."
-us,scenario_008,payroll_tax,gemini-4-argon,llm_error,state_local_rule,False,It stopped at 7.65% federal FICA. It never added the $226.46 of NJ employee state payroll contributions required by the output definition.
-us,scenario_008,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It returned no payroll_tax value or explanation, so there was no answer to score against $2,276.66."
-us,scenario_008,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It explicitly treated NJ employee state payroll tax as not modeled. It omitted the $226.46 of TDI, FLI, UI and workforce contributions."
-us,scenario_008,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It correctly computed $1,661.60 Social Security plus $388.60 Medicare, but then reported an unsupported $4,097, roughly double that sum. It also omitted the $226.46 of NJ employee contributions."
-us,scenario_008,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"Its $1,525 is less than even the federal FICA on $26,800 ($2,050.20), so it misapplied the 7.65% rate to the wage base. It also left out the $226.46 of NJ employee contributions."
-us,scenario_008,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It applied a combined NJ employee rate of 0.9425% ($252.59). The 2026 rates are UI 0.3825% + WF 0.0425% + TDI 0.19% + FLI 0.23% = 0.845% ($226.46), so it overstated the NJ contributions by $26.13."
-us,scenario_008,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"It put NJ employee contributions at $253.26 (about 0.945%) instead of $226.46 (0.845%). It overstated the combined TDI and FLI rates by about 0.10 percentage points, $26.80 too much."
-us,scenario_008,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $263.98 equals 0.985% of wages, matching the 2025 worker rates of TDI 0.23% and FLI 0.33%. The 2026 rates are TDI 0.19% and FLI 0.23%, which give $226.46."
-us,scenario_008,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,It put NJ employee contributions at $202.34 (0.755%) instead of $226.46 (0.845%). It understated the combined TDI (0.19%) and FLI (0.23%) contributions by $24.12.
-us,scenario_008,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"Its $263.98 equals 0.985% of wages, matching the 2025 worker rates of TDI 0.23% and FLI 0.33% instead of the 2026 rates of 0.19% and 0.23%. It overstated the NJ contributions by $37.52."
-us,scenario_008,payroll_tax,gpt-6.1-sol,llm_error,state_local_rule,False,"Its $215.07 equals 0.8025% of wages: UI 0.3825% + TDI 0.19% + FLI 0.23%. It left out the separate 0.0425% Workforce/SWF contribution ($11.39), which is exactly the gap to the reference."
-us,scenario_008,payroll_tax,gpt-6.1-terra,llm_error,state_local_rule,False,"Its $237.18 equals 0.885% of wages, consistent with the 2025 TDI rate of 0.23% instead of the 2026 rate of 0.19%. That overstated NJ contributions by $10.72."
-us,scenario_008,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only 7.65% FICA (submitted as $2,050). It omitted the $226.46 of mandatory NJ employee TDI/FLI/UI/workforce contributions."
-us,scenario_008,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It counted only TDI and FLI at a combined 0.18% ($48.24) instead of 0.42% ($112.56). It also omitted the NJ employee UI (0.3825%) and workforce (0.0425%) contributions, together $113.90."
-us,scenario_008,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It added only about $178 of NJ contributions, made up of UI at 0.3825% plus an underestimated TDI/FLI amount. It left out the 0.0425% workforce contribution and fell short of the full $112.56 of TDI (0.19%) plus FLI (0.23%)."
-us,scenario_008,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"It explicitly used TDI 0.23% and FLI 0.33%, which are the 2025 worker rates. The 2026 rates are 0.19% and 0.23%, so it overstated NJ contributions by $37.52."
-us,scenario_008,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It explicitly assumed there were no state payroll taxes. New Jersey imposes mandatory employee TDI, FLI, UI and workforce contributions totaling $226.46 on these wages."
-us,scenario_008,payroll_tax,inkling,llm_error,state_local_rule,False,"It counted only NJ UI (0.3825%) and FLI at 0.09%. It omitted worker TDI (0.19%, $50.92) and the workforce contribution (0.0425%, $11.39), and it understated FLI, whose 2026 rate is 0.23%."
-us,scenario_008,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no payroll_tax value or explanation, so there was no answer to score against $2,276.66."
-us,scenario_008,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"It put NJ UI/workforce, TDI and FLI at $257.95 (about 0.96%) instead of $226.46 (0.845%). It overstated the 2026 TDI (0.19%) and FLI (0.23%) worker rates."
-us,scenario_008,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It counted only federal Social Security and Medicare, rounded to $2,050. It omitted the $226.46 of mandatory NJ employee contributions."
-us,scenario_008,payroll_tax,muse-spark-1.3,llm_error,state_local_rule,False,"It estimated NJ UI/TDI/FLI at about $241 instead of $226.46, which comes from 0.845% of $26,800 using the 2026 TDI rate of 0.19% and FLI rate of 0.23%. That overshot the total by about $14."
-us,scenario_008,payroll_tax,muse-wonder,llm_error,state_local_rule,False,"It approximated NJ employee contributions at roughly $241 instead of the exact $226.46, which comes from UI 0.3825%, WF 0.0425%, TDI 0.19% and FLI 0.23%. Rounding FICA up to $2,051 pushed the total to $2,292."
-us,scenario_008,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It assumed no mandatory NJ employee payroll tax was modeled. It omitted the $226.46 of TDI, FLI, UI and workforce contributions."
-us,scenario_008,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It included only NJ UI at 0.3825% ($102.51) and set worker DI and FLI to 0% using 2024 assumptions. The 2026 worker rates are TDI 0.19% ($50.92) and FLI 0.23% ($61.64), and it also omitted the 0.0425% workforce contribution ($11.39)."
-us,scenario_008,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It applied only 7.65% FICA. It omitted New Jersey's mandatory employee TDI, FLI, UI and workforce contributions, which total $226.46."
+us,scenario_008,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"It counted NJ UI ($102.51) and workforce fund ($11.39) correctly. However, it set the 2026 employee TDI rate to 0% instead of 0.19% ($50.92) and used 0.09% for FLI instead of 0.23% ($61.64), which left the NJ total $97.81 short."
+us,scenario_008,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It reported only federal FICA ($2,050.20). It left out NJ's mandatory employee TDI (0.19%), FLI (0.23%), UI (0.3825%) and workforce fund (0.0425%) contributions, which add $226.46 on the $26,800 of wages."
+us,scenario_008,payroll_tax,claude-fable-5.2,llm_error,state_local_rule,False,"It explicitly assumed NJ employee payroll contributions are not included, even though the output definition lists mandatory employee state payroll taxes. As a result it dropped the $226.46 of NJ TDI, FLI, UI and workforce-fund contributions."
+us,scenario_008,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It computed a 0.765% NJ UI figure and then discarded it, substituting an invented $2.12 'minimum assessment.' The actual NJ employee contributions are TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce fund 0.0425% on $26,800, totaling $226.46."
+us,scenario_008,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"It stopped at federal FICA of $2,050.20 and never added NJ's mandatory employee TDI, FLI, UI and workforce-fund contributions, which come to $226.46."
+us,scenario_008,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It recognized NJ's UI, SDI and FLI employee taxes but dropped all of them because of rate uncertainty, and it wrongly took the 2026 employee TDI rate to be 0%. The correct amounts on $26,800 are TDI $50.92, FLI $61.64, UI $102.51 and workforce fund $11.39."
+us,scenario_008,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It wrongly assumed NJ's mandatory UI, SDI and FLI employee contributions are not modeled and set them to 0. That left out $226.46 of NJ employee payroll tax."
+us,scenario_008,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It explicitly excluded NJ employee UI, TDI and FLI from the measure, even though the definition includes mandatory employee state payroll taxes. It also rounded FICA to $2,050, missing the $226.46 NJ component."
+us,scenario_008,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It applied only the 7.65% federal FICA rate and left out NJ's mandatory employee TDI (0.19%), FLI (0.23%), UI (0.3825%) and workforce fund (0.0425%) contributions, which total $226.46."
+us,scenario_008,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It wrongly claimed NJ TDI is employer-funded with a 0% employee rate and treated FLI as outside 'payroll tax,' so it used federal FICA only. In fact NJ workers pay TDI at 0.19%, FLI at 0.23%, UI at 0.3825% and workforce fund at 0.0425%, adding $226.46."
+us,scenario_008,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It estimated NJ employee UI/DI/FLI at about $201–$210 but then zeroed it out over 'cap uncertainty.' The spouse's $26,800 is below every NJ wage base, so the full $226.46 applies."
+us,scenario_008,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It wrongly asserted that NJ employee payroll taxes are not modeled and left out the $226.46 of TDI, FLI, UI and workforce-fund contributions on the spouse's wages."
+us,scenario_008,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It computed only federal FICA and rounded it down to $2,050, leaving out the $226.46 of NJ mandatory employee TDI, FLI, UI and workforce-fund contributions."
+us,scenario_008,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It used outdated rates of 0.10% for TDI ($26.80 instead of $50.92) and 0.09% for FLI ($24.12 instead of $61.64). It also left out NJ's UI (0.3825%, $102.51) and workforce fund (0.0425%, $11.39) employee contributions entirely."
+us,scenario_008,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Its NJ employee payroll tax of $154.10 implies a combined rate of 0.575% instead of the correct 0.845% (TDI 0.19% + FLI 0.23% + UI 0.3825% + workforce fund 0.0425%), understating the NJ total by $72.36."
+us,scenario_008,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It computed only Social Security and Medicare ($2,050.20) and never added NJ's mandatory employee TDI, FLI, UI and workforce-fund contributions of $226.46."
+us,scenario_008,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied a combined 0.515% for NJ SUI and FLI: UI and workforce fund at 0.425% plus FLI at an outdated 0.09%. That understated FLI (0.23% = $61.64) and left out TDI entirely (0.19% = $50.92). It also rounded the total down to $2,188."
+us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It applied only 7.65% FICA and rounded it to $2,050, leaving out NJ's mandatory employee TDI, FLI, UI and workforce-fund contributions totaling $226.46."
+us,scenario_008,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Its roughly $138 NJ estimate matches UI and workforce fund (0.425%) plus FLI at 0.09%, effectively leaving out TDI at 0.19% ($50.92) and understating FLI, which is 0.23% ($61.64). The correct NJ total is $226.46."
+us,scenario_008,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It counted only NJ UI and FLI, at $123.80, leaving out TDI (0.19%, $50.92) and understating FLI, which is 0.23% ($61.64). The full NJ employee total is $226.46."
+us,scenario_008,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It applied only 7.65% FICA to the $26,800 of wages and left out NJ's $226.46 of mandatory employee TDI, FLI, UI and workforce-fund contributions."
+us,scenario_008,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"It counted only NJ SUI and FLI at 0.515% ($138.02), using an outdated 0.09% FLI rate instead of 0.23%. It also left out the 0.19% employee TDI contribution ($50.92), undercounting the NJ portion by $88.44."
+us,scenario_008,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"It included NJ FLI, SUI and workforce fund but left out the 0.19% employee TDI contribution ($50.92) and used too low an FLI rate. Its NJ total was $139.32 instead of $226.46."
+us,scenario_008,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It stopped at Social Security and Medicare ($2,050.20) and never added NJ's mandatory employee TDI, FLI, UI and workforce-fund contributions of $226.46."
+us,scenario_008,payroll_tax,gemini-4-argon,llm_error,state_local_rule,False,"It treated payroll tax as 7.65% FICA only and left out the $226.46 of NJ mandatory employee contributions (TDI 0.19%, FLI 0.23%, UI 0.3825%, workforce fund 0.0425%)."
+us,scenario_008,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value. The correct derivation is $2,050.20 of FICA on the spouse's $26,800 wages plus $226.46 of NJ employee TDI, FLI, UI and workforce-fund contributions, for $2,276.66."
+us,scenario_008,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It wrongly assumed no NJ employee state payroll tax is modeled and left out the $226.46 of TDI, FLI, UI and workforce-fund contributions."
+us,scenario_008,payroll_tax,gpt-5.4-mini,llm_error,other,False,"Its own components summed to $2,050.20, but it submitted $4,097, which is roughly double FICA and not supported by its reasoning. It also wrongly said no NJ employee payroll tax applies, missing the $226.46 of TDI, FLI, UI and workforce-fund contributions."
+us,scenario_008,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $1,525 is below even the 7.65% FICA amount of $2,050.20 on $26,800, which means it misapplied the Social Security and Medicare rates. It also left out the $226.46 of NJ mandatory employee contributions."
+us,scenario_008,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,It used a combined NJ employee rate of 0.9425% instead of 0.845% (TDI 0.19% + FLI 0.23% + UI 0.3825% + workforce fund 0.0425%). That overstated the TDI and FLI rates and inflated the NJ contributions by $26.13.
+us,scenario_008,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"Its NJ contributions of $253.26 (0.945%) are 0.10 percentage point too high. That matches using the prior 0.33% FLI rate instead of the 2026 rate of 0.23%, giving $88.44 of FLI instead of $61.64."
+us,scenario_008,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"Its NJ total of $263.98 (0.985%) uses prior-year rates of 0.23% for TDI and 0.33% for FLI. The 2026 rates are 0.19% and 0.23%, so NJ contributions were overstated by $37.52."
+us,scenario_008,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Its NJ total of $202.34 (0.755%) is $24.12 below the correct $226.46. The correct total comes from TDI 0.19%, FLI 0.23%, UI 0.3825% and workforce fund 0.0425% on $26,800, so it misstated the TDI and FLI rates."
+us,scenario_008,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"Its NJ total of $263.98 (0.985%) uses outdated rates of 0.23% for TDI and 0.33% for FLI instead of the 2026 rates of 0.19% and 0.23%, overstating NJ contributions by $37.52."
+us,scenario_008,payroll_tax,gpt-6.1-sol,llm_error,state_local_rule,False,Its NJ total of $215.07 equals 0.8025% (UI 0.3825% + TDI 0.19% + FLI 0.23%). It treated the 0.3825% UI rate as already including the workforce fund and so missed the separate 0.0425% workforce-fund contribution ($11.39).
+us,scenario_008,payroll_tax,gpt-6.1-terra,llm_error,state_local_rule,False,"Its NJ total of $237.18 (0.885%) is $10.72 too high. That matches using the prior 0.23% TDI rate instead of the 2026 rate of 0.19%, together with FLI at 0.23% and UI plus workforce fund at 0.425%."
+us,scenario_008,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only 7.65% FICA (rounded to $2,050) and left out NJ's mandatory employee TDI, FLI, UI and workforce-fund contributions, which total $226.46."
+us,scenario_008,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It counted only TDI and FLI, at a combined 0.18% ($48.24) instead of 0.42% ($112.56), and it left out NJ's employee UI (0.3825%, $102.51) and workforce fund (0.0425%, $11.39) entirely."
+us,scenario_008,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It estimated NJ UI plus TDI/FLI at about $178 instead of $226.46 by understating the TDI (0.19%) and FLI (0.23%) amounts and leaving out the separate workforce fund. It also rounded FICA to $2,050."
+us,scenario_008,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"It used prior-year rates of 0.23% for TDI and 0.33% for FLI instead of the 2026 rates of 0.19% and 0.23%. That gave 0.985% instead of 0.845%, overstating NJ contributions by $37.52."
+us,scenario_008,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It wrongly stated there are no state payroll taxes. NJ requires employee TDI, FLI, UI and workforce-fund contributions totaling $226.46 on $26,800 of wages."
+us,scenario_008,payroll_tax,inkling,llm_error,state_local_rule,False,"It included only UI at 0.3825% and FLI at an outdated 0.09%. It left out TDI (0.19%, $50.92) and the workforce fund (0.0425%, $11.39) and understated FLI, which is 0.23% ($61.64)."
+us,scenario_008,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value. The correct derivation is $2,050.20 of FICA plus $226.46 of NJ employee TDI, FLI, UI and workforce-fund contributions, for $2,276.66."
+us,scenario_008,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"Its NJ total of $257.95 (0.9625%) overstates the combined TDI and FLI rate. Those rates are 0.19% and 0.23% for 2026, so the correct NJ total is $226.46 (0.845%), and it was $31.49 too high."
+us,scenario_008,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It computed only federal FICA, rounded to $2,050, and never added NJ's mandatory employee TDI, FLI, UI and workforce-fund contributions of $226.46."
+us,scenario_008,payroll_tax,muse-spark-1.3,llm_error,state_local_rule,False,"It estimated NJ UI/TDI/FLI at about $241 instead of $226.46, overstating the combined employee rate relative to 0.845% (TDI 0.19%, FLI 0.23%, UI 0.3825%, workforce fund 0.0425%). It also rounded FICA to $2,050."
+us,scenario_008,payroll_tax,muse-wonder,llm_error,state_local_rule,False,"It approximated NJ unemployment, disability and family-leave contributions at about $241 instead of $226.46, overstating the 2026 TDI and FLI rates (0.19% and 0.23%). It also rounded the federal components."
+us,scenario_008,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It wrongly assumed NJ employee payroll tax is not modeled and left out the $226.46 of TDI, FLI, UI and workforce-fund contributions on the spouse's $26,800 wages."
+us,scenario_008,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It used 2024 rates that set employee TDI and FLI to 0%, so it counted only UI at 0.3825% ($102.51). It missed the 2026 TDI (0.19%, $50.92), FLI (0.23%, $61.64) and workforce-fund (0.0425%, $11.39) contributions."
+us,scenario_008,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It treated employee payroll tax as 7.65% FICA only and left out NJ's mandatory employee TDI, FLI, UI and workforce-fund contributions totaling $226.46."
us,scenario_008,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the reduced-price ceiling and housing assistance as sufficient for reduced-price support, without applying the mutually exclusive free-meals tier. At 55% of the federal poverty guideline and with categorical eligibility through SNAP/TANF, the household qualifies for free meals, which makes reduced-price eligibility false."
us,scenario_008,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, so its response failed the required output contract."
us,scenario_008,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model returned $306, exactly 7.65% of $4,000, despite stating that it applied 15.3%. It effectively used only half the combined Social Security and Medicare self-employment tax rate and also failed to apply the 92.35% statutory earnings factor, instead of calculating $4,000 × 0.9235 × 0.153 = $565.18."
us,scenario_008,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, so its response failed the required output contract."
us,scenario_008,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model correctly calculated gross self-employment tax of $565.18, then incorrectly subtracted a supposed deduction from that tax liability. The deductible half of self-employment tax reduces adjusted gross income for income-tax purposes; it is not a credit against self-employment tax, so the liability remains $565.18."
-us,scenario_008,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used a $258 standard deduction instead of $299 and the outdated FY2025 $1,756 maximum allotment instead of $1,789. It then dropped its own $1,214/month result and switched to an unsupported $687/month. The correct figures give $1,789 − $530 = $1,259 a month."
-us,scenario_008,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"It used a $302 standard deduction instead of the FY2026 6+ person figure of $299, which gave net income of $1,760.92. It also skipped SNAP rounding: net income should be rounded to $1,764 and the 30% contribution rounded up to $530, for a benefit of $1,259 rather than $1,260.72."
-us,scenario_008,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It left out the 20% earned income deduction ($513.33) and used a $227 standard deduction instead of $299, which overstated net income at $2,349. It also used a far-too-low $1,164 maximum allotment instead of $1,789."
-us,scenario_008,snap,claude-haiku-5.5,llm_error,asset_resource,False,"It applied a SNAP asset limit to the $69,500 in bank assets. In New Jersey the household is categorically eligible through TANF non-cash benefits (broad-based categorical eligibility), which removes the asset test, so the household gets $1,259 a month."
-us,scenario_008,snap,claude-opus-4.7,llm_error,period_annualization,False,"It subtracted the monthly standard deduction (~$280) once from annual net income ($24,755) instead of $299 × 12, which inflated monthly net income to $2,040 instead of $1,764. It also used the FY2025 $1,756 maximum allotment instead of $1,789."
-us,scenario_008,snap,claude-opus-4.8,llm_error,period_annualization,False,"It subtracted the $258 monthly standard deduction once from annual income instead of $299 every month, which inflated monthly net income to $2,041 instead of $1,764. It also used the FY2025 $1,756 maximum, then cut its own $13,728 result to an arbitrary $11,352."
-us,scenario_008,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It assumed a shelter deduction cut net income to about $1,700, but the $579 rent is below half of net income, so the shelter deduction is $0 and net income is $1,764. Its stated $1,040 benefit also doesn't follow from its own $1,802 maximum. The correct benefit is $1,789 − $530 = $1,259."
-us,scenario_008,snap,claude-opus-5.5,llm_error,other,False,"Its inputs were right ($299 standard deduction, $1,789 maximum), but it skipped SNAP rounding. Net income $1,763.92 rounds to $1,764, and 30% of that ($529.20) rounds up to a $530 contribution, so the benefit is $1,259 a month, not $1,259.82."
-us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $1,536 maximum allotment for 8 people instead of the FY2026 figure of $1,789, and a $258 standard deduction instead of $299. That gave $995 a month instead of $1,259."
-us,scenario_008,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It made up a shelter deduction that included a utility allowance even though no utility costs were listed, and used a $204 standard deduction. That pushed net income down to about $733 a month instead of $1,764. Its final $958 a month also doesn't follow from its own $1,691 maximum."
-us,scenario_008,snap,claude-sonnet-5.5,llm_error,other,False,"It rounded the 30% contribution down to $529. SNAP rounds 30% of the $1,764 net income ($529.20) up to $530, so the benefit is $1,259 a month, not $1,260."
-us,scenario_008,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a ~$1,780 maximum instead of $1,789 and overstated net income at ~$1,830 instead of $1,764, which implies a standard deduction below $299. It also invoked a shelter deduction that doesn't apply because rent is below half of net income."
-us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a ~$1,900 maximum allotment for 8 people instead of the FY2026 figure of $1,789, and a $302 standard deduction instead of $299. That overstated the benefit at $1,371 a month instead of $1,259."
-us,scenario_008,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the FY2025 $1,756 maximum allotment instead of $1,789. Its $1,783.92 net income implies a ~$279 standard deduction instead of $299. That understated the benefit at $1,221 a month instead of $1,259."
-us,scenario_008,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $1,800 maximum allotment instead of $1,789. Its $1,752.92 net income implies a ~$310 standard deduction instead of $299. That overstated the benefit at $1,274 a month instead of $1,259."
-us,scenario_008,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It adjusted for a shelter deduction, but the $579 rent is below half of the $1,764 net income, so the shelter deduction is $0. Its $1,275 a month is $16 above the correct $1,789 − $530 = $1,259."
-us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no calculation. Its $16,400 a year ($1,367 a month) implies a contribution of only ~$422, or an inflated maximum allotment. The correct figures are $1,764 net income, a $530 contribution, and $1,259 a month."
-us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $1,307 a month implies a contribution of about $482, which means net income near $1,607 or a maximum above $1,789. The correct net income after the $299 standard and $513.33 earned income deductions is $1,764, giving a $530 contribution and $1,259 a month."
-us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It added New Jersey's standard utility allowance to rent to create an excess shelter deduction, but no utility costs were listed. Rent alone ($579) is below half of net income, so there is no shelter deduction and the benefit is $1,259 a month, not ~$1,400."
-us,scenario_008,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation. Its $1,208 a month is $51 below the correct $1,259 ($1,789 − $530)."
-us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $287 standard deduction instead of FY2026's $299 for 6+ person households, and the FY2025 $1,756 maximum instead of $1,789. That gave $1,223 a month instead of $1,259."
-us,scenario_008,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no calculation. Its $1,404 a month implies a contribution of only $385 against a $1,789 maximum, far below the $530 that 30% of the $1,764 net income requires."
-us,scenario_008,snap,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It included a shelter deduction that doesn't apply, since the $579 rent is below half of the $1,764 net income. That cut countable income and inflated the benefit to $1,317 a month instead of $1,259."
-us,scenario_008,snap,gemini-4-argon,llm_error,other,False,"Its $1,262 a month is $3 above the correct $1,259. That is consistent with skipping SNAP rounding (net income rounded to $1,764, 30% contribution rounded up to $530) and using a slightly higher standard deduction."
-us,scenario_008,snap,glm-5.2,llm_error,period_annualization,False,"It subtracted a $198 monthly standard deduction once from annual income instead of $299 × 12, which left annual net income at $24,557 ($2,046 a month) instead of $1,764 a month. It also used a $1,748 maximum instead of $1,789."
-us,scenario_008,snap,glm-5.3,llm_error,thresholds_rates,False,"It used a $210 standard deduction instead of FY2026's $299 for 6+ person households, which raised net income to $1,853, and a $1,797 maximum instead of $1,789. That gave $1,241 a month instead of $1,259."
-us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It assumed housing assistance and combined earnings would leave the household ineligible. Gross income is only 57% of the poverty guideline, the household is categorically eligible in NJ, and net income of $1,764 yields $1,259 a month."
-us,scenario_008,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It answered $0 on the claim that eligibility inputs were missing. The listed wages, self-employment income, interest, and 8-person household size fully determine a $1,259 monthly benefit using the standard and earned income deductions."
-us,scenario_008,snap,gpt-5.5,llm_error,other,False,"It reached $1,260 a month by rounding the 30% contribution down to $529. SNAP rounds 30% of the $1,764 net income up to $530, giving $1,259 a month."
-us,scenario_008,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $1,139 a month is $120 below the correct benefit, which means an outdated maximum allotment or overstated countable income. No shelter deduction applies, and the correct calculation is $1,789 − 30% of $1,764 (rounded up to $530) = $1,259."
-us,scenario_008,snap,gpt-5.6-sol,llm_error,other,False,"Its $1,262 a month is $3 above the correct $1,259. It didn't apply SNAP rounding exactly: net income $1,763.92 rounds to $1,764, and 30% ($529.20) rounds up to a $530 contribution."
-us,scenario_008,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $1,284 a month implies a contribution of only $505, which means net income near $1,683 or a maximum above $1,789. The correct net income after the $299 standard and $513.33 earned income deductions is $1,764, giving $1,259 a month."
-us,scenario_008,snap,gpt-6-astra,llm_error,other,False,"It annualized unrounded figures ($21,468 − 0.3 × $21,167). SNAP is computed monthly: net income rounds to $1,764 and the contribution rounds up to $530, so the benefit is $1,259 × 12 = $15,108."
-us,scenario_008,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $1,288 a month implies a ~$501 contribution, which understates net income or overstates the maximum. The FY2026 $299 standard deduction and $1,789 maximum give net income of $1,764, a $530 contribution, and $1,259 a month."
-us,scenario_008,snap,gpt-6-sol,llm_error,other,False,"It subtracted a $529 contribution from $1,789. SNAP rounds 30% of the $1,764 net income ($529.20) up to $530, so the benefit is $1,259 a month, not $1,260."
-us,scenario_008,snap,gpt-6.1-terra,llm_error,thresholds_rates,False,"Its $1,265 a month is $6 above the correct benefit, which is consistent with a maximum allotment above $1,789 or a standard deduction above $299. The correct calculation is $1,789 − $530 = $1,259."
-us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"It wrongly concluded that housing assistance and income rule out SNAP. Gross income is 57% of the poverty guideline, the household passes both income tests with categorical eligibility, and receives $1,259 a month."
-us,scenario_008,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"It took an excess medical deduction from OTC expenses, but the disabled spouse's $300 a year ($25 a month) is below the $35 monthly threshold, so the medical deduction is $0. It also used a ~$1,822.50 monthly maximum instead of $1,789."
-us,scenario_008,snap,grok-4.6,llm_error,thresholds_rates,False,"It used the FY2025 standard deduction (~$291), giving net income of $1,772 instead of $1,764, and a $1,759 maximum instead of FY2026's $1,789. That gave $1,227 a month instead of $1,259."
-us,scenario_008,snap,grok-4.7,llm_error,thresholds_rates,False,"It explicitly used FY2025 parameters: a $1,756 maximum ($21,072 a year) and a $291 standard deduction ($3,492 a year). The FY2026 figures that apply are $1,789 and $299, which give $1,259 a month."
-us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"It applied the $4,250 resource limit for households with a disabled member. NJ's broad-based categorical eligibility through TANF non-cash benefits removes the asset test, so the $69,500 balance doesn't matter and the benefit is $1,259 a month."
-us,scenario_008,snap,inkling,llm_error,thresholds_rates,False,"It used a $1,806 maximum allotment for 8 people instead of FY2026's $1,789. That gave about $1,276 a month instead of $1,789 − $530 = $1,259."
-us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no SNAP value and no explanation, so there was nothing to parse. The correct calculation yields $1,259 a month ($15,108 a year)."
-us,scenario_008,snap,kimi-k3,llm_error,other,False,"Its inputs were right, but it skipped SNAP rounding. Net income $1,763.92 rounds to $1,764 and the contribution rounds up to $530, so the benefit is $1,259 a month, not $1,259.825."
-us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"It applied a $3,750 asset limit and found the household ineligible. New Jersey's TANF non-cash categorical eligibility removes the SNAP asset test, so the household receives $1,259 a month."
-us,scenario_008,snap,muse-spark-1.3,llm_error,thresholds_rates,False,"Its ~$1,300 a month exceeds the correct benefit by $41, which implies an inflated maximum or understated net income. The $299 standard deduction, $513.33 earned income deduction, and $1,789 maximum give $1,259."
-us,scenario_008,snap,muse-wonder,llm_error,thresholds_rates,False,"It used a $1,695 maximum allotment instead of FY2026's $1,789. Its $1,772 net income implies the FY2025 ~$291 standard deduction instead of $299. That gave $1,163 a month instead of $1,259."
-us,scenario_008,snap,ox-alpha,llm_error,thresholds_rates,False,"It used a $1,795 maximum instead of $1,789 and a $297 standard deduction instead of $299. That gave a $1,265 monthly benefit instead of $1,789 − $530 = $1,259."
-us,scenario_008,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $202 standard deduction instead of $299 and a $1,838 maximum instead of $1,789. It then cut its own $15,391 result to an arbitrary $14,784 for housing assistance, even though no shelter deduction applies either way."
-us,scenario_008,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It claimed $2,567 a month exceeds 130% of the poverty line for 8 people. In fact the 8-person guideline is far higher: the household's gross income is only 57% of poverty, so it passes the gross income test and receives $1,259 a month."
+us,scenario_008,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly got $1,214/month using FY2025 figures ($258 standard deduction, $1,756 maximum). It then replaced that with an unexplained $687/month 'estimate revised'. The correct FY2026 figures are a $299 standard deduction and a $1,789 maximum, which give net income of $1,764 and a benefit of $1,259/month."
+us,scenario_008,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"The model used a $302 standard deduction instead of the FY2026 $299 for households of 6 or more. It also skipped SNAP rounding: net income must round to a whole dollar ($1,764) and the 30% contribution rounds up ($530). Together these gave $1,260.72/month instead of $1,259."
+us,scenario_008,snap,claude-fable-5.2,llm_error,other,False,"Every input was right ($299 standard deduction, $1,789 maximum, net income $1,763.92), but the model never rounded. Net income must round to $1,764, and 30% of that ($529.20) rounds up to a $530 contribution, giving $1,259/month instead of $1,259.83."
+us,scenario_008,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model left out the 20% earned income deduction ($513.33) and used a $227 standard deduction instead of $299. It also used a $1,164 maximum allotment for 8 people, far below the FY2026 figure of $1,789. Its $583/month result does not even follow from its own numbers."
+us,scenario_008,snap,claude-haiku-5.5,llm_error,asset_resource,False,"The model denied SNAP because the $69,500 bank balance is over the resource limit. NJ confers categorical eligibility through TANF non-cash benefits (broad-based categorical eligibility), which removes the asset test, so the household gets $1,259/month."
+us,scenario_008,snap,claude-opus-4.7,llm_error,period_annualization,False,"The model subtracted the monthly standard deduction (~$280) only once from annual net income, instead of subtracting it in every month. That left net income at $2,040/month instead of $1,764. It also used the FY2025 maximum of $1,756 instead of $1,789, then cut its own result to $13,620 without explanation."
+us,scenario_008,snap,claude-opus-4.8,llm_error,period_annualization,False,"The model subtracted a $258 standard deduction once from annual income, which inflated net income to $2,041/month instead of $1,764. It also used the FY2025 maximum of $1,756 instead of $1,789. It then made an unexplained downward 'adjustment' from $13,728 to $11,352."
+us,scenario_008,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model claimed an excess shelter deduction that does not exist here, because $579 rent is below half of adjusted income. It also used a $1,802 maximum instead of $1,789. Its stated $1,040/month does not follow from its own $1,700 net income figure."
+us,scenario_008,snap,claude-opus-5.5,llm_error,other,False,"The model used the correct $299 deduction and $1,789 maximum but did not round net income to $1,764 or round the 30% contribution up to $530. That gives $1,259.82/month instead of $1,259."
+us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $1,536 maximum allotment for 8 people instead of FY2026's $1,789. It also used a $258 standard deduction instead of $299 for households of 6 or more. The result was net income of $1,805 and a benefit of $995/month instead of $1,259."
+us,scenario_008,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model added a utility allowance and an excess shelter deduction that the household lacks, because no utility costs are listed and $579 rent is below half of adjusted income. That drove net income down to about $733/month. It also used a $204 standard deduction and a $1,691 maximum instead of $299 and $1,789."
+us,scenario_008,snap,claude-sonnet-5.5,llm_error,other,False,"The model correctly rounded net income to $1,764 but treated the contribution as roughly $529. SNAP rounds 30% of net income ($529.20) up to $530, so the benefit is $1,259/month, not $1,260."
+us,scenario_008,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model put net income at about $1,830 instead of $1,764, which understates the $299 standard deduction. It also used a maximum of about $1,780 instead of $1,789 and mentioned shelter deductions that do not apply. The result was $1,225/month instead of $1,259."
+us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a maximum allotment of about $1,900 for 8 people instead of FY2026's $1,789, and a $302 standard deduction instead of $299. That overstated the benefit at $1,371/month instead of $1,259."
+us,scenario_008,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used the FY2025 maximum of $1,756 instead of FY2026's $1,789. Its net income of $1,783.92 implies a $279 standard deduction instead of $299. The result was $1,220.83/month instead of $1,259."
+us,scenario_008,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used an $1,800 maximum allotment instead of $1,789. Its net income of $1,752.92 implies a $310 standard deduction instead of $299. It also skipped rounding, giving $1,274.12/month instead of $1,259."
+us,scenario_008,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model claimed 'shelter deductions' that do not apply, since rent is below half of adjusted income. Its $15,300 total is $1,275/month, $16 above the correct $1,789 − $530 = $1,259."
+us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model gave no computation. Its $16,400 total is about $1,367/month, $108 above the correct $1,259 ($1,789 maximum minus 30% of $1,764 net income, rounded up to $530). That is consistent with an inflated maximum allotment or a skipped earned income or standard deduction offset."
+us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's $1,307/month is $48 above the correct $1,259. That matches using a maximum allotment of about $1,837 instead of FY2026's $1,789 against the $530 contribution from $1,764 net income."
+us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied an excess shelter deduction using NJ's standard utility allowance, even though no utility expenses are listed. Without the allowance, $579 rent is below half of adjusted income, so there is no shelter deduction. That error inflated the benefit to about $1,400/month instead of $1,259."
+us,scenario_008,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no computation. Its $14,500 total is about $1,208/month, $51 below the correct $1,259 ($1,789 maximum minus a $530 contribution on $1,764 net income). That is consistent with using an outdated maximum allotment near FY2025's $1,756 or a smaller standard deduction."
+us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $287 standard deduction instead of FY2026's $299 for households of 6 or more, and the FY2025 maximum of $1,756 instead of $1,789. The result was $1,223/month instead of $1,259."
+us,scenario_008,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $16,848 total is $1,404/month, $145 above the correct $1,259. That is consistent with an inflated maximum allotment or with understating net income below the correct $1,764."
+us,scenario_008,snap,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted a shelter deduction, but rent of $579 is below half of adjusted income and no utilities are listed, so the household has none. That overstated the benefit at about $1,317/month instead of $1,259."
+us,scenario_008,snap,gemini-4-argon,llm_error,other,False,"The model's $15,144 total is $1,262/month, $3 above the correct $1,259. That is consistent with a contribution of $527 instead of 30% of the rounded $1,764 net income ($529.20), rounded up to $530."
+us,scenario_008,snap,glm-5.2,llm_error,period_annualization,False,"The model subtracted a $198 standard deduction once from annual income instead of $299 every month, which inflated net income to $24,557/yr. It also used a $1,748 maximum instead of $1,789. The result was $13,608.90 instead of $15,108."
+us,scenario_008,snap,glm-5.3,llm_error,thresholds_rates,False,"The model used a $210 standard deduction instead of FY2026's $299 for households of 6 or more, which put net income at $1,853 instead of $1,764. It also used a $1,797 maximum instead of $1,789."
+us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model returned zero without computing anything. The household is categorically eligible through NJ TANF non-cash benefits, and gross income is only 57% of poverty. Net income of $1,764 gives a $1,259/month benefit."
+us,scenario_008,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model refused to compute because inputs were 'missing', but every needed input was listed: wages, self-employment and interest income, and household size. Those yield net income of $1,764 and a $1,259/month benefit under NJ categorical eligibility."
+us,scenario_008,snap,gpt-5.5,llm_error,other,False,"The model's method was right, but its contribution of about $529 was not rounded up. SNAP rounds 30% of the $1,764 net income ($529.20) up to $530, giving $1,259/month instead of $1,260."
+us,scenario_008,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model claimed a shelter deduction that does not exist, since rent is below half of adjusted income. Its $1,139/month is $120 below the correct $1,259. That is consistent with leaving out the $299 standard deduction (net about $2,063) and using the FY2025 $1,756 maximum."
+us,scenario_008,snap,gpt-5.6-sol,llm_error,other,False,"The model's $15,144 total is $1,262/month, $3 above the correct $1,259. That implies a contribution of about $527 instead of 30% of the rounded $1,764 net income, rounded up to $530."
+us,scenario_008,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model's $15,408.20 total is $1,284.02/month, $25 above the correct $1,789 − $530 = $1,259. That is consistent with an inflated maximum allotment of about $1,814 or with understating net income below $1,764."
+us,scenario_008,snap,gpt-6-astra,llm_error,period_annualization,False,"The model computed annually as $21,468 − 0.3 × $21,167 and skipped SNAP's monthly rounding. Net income must round to $1,764 each month and the contribution round up to $530, which gives $1,259 × 12 = $15,108."
+us,scenario_008,snap,gpt-6-luna,llm_error,thresholds_rates,False,"The model's $15,458 total is about $1,288/month, $29 above the correct $1,259. That is consistent with using a maximum allotment of about $1,818 instead of FY2026's $1,789 against the $530 contribution."
+us,scenario_008,snap,gpt-6-sol,llm_error,other,False,"The model truncated net income to $1,763 instead of rounding $1,763.92 to $1,764, and used a contribution of about $529. SNAP rounds 30% of $1,764 ($529.20) up to $530, so the benefit is $1,259/month, not $1,260."
+us,scenario_008,snap,gpt-6.1-terra,llm_error,thresholds_rates,False,"The model's $15,180 total is $1,265/month, $6 above the correct $1,259. That matches a maximum allotment of $1,795 instead of FY2026's $1,789 minus the $530 contribution."
+us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model claimed housing assistance plus income leaves no SNAP. Housing assistance does not disqualify a household. With gross income at 57% of poverty and NJ categorical eligibility, net income of $1,764 yields $1,259/month."
+us,scenario_008,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model applied an excess medical deduction from the disabled spouse's over-the-counter expenses. Those are $25/month, below the $35 threshold, so the deduction is zero. It also used a maximum of about $1,822/month instead of $1,789."
+us,scenario_008,snap,grok-4.6,llm_error,thresholds_rates,False,"The model used a maximum of about $1,759 instead of FY2026's $1,789. Its net income of $1,772 implies a $291 standard deduction instead of $299. The result was $1,227/month instead of $1,259."
+us,scenario_008,snap,grok-4.7,llm_error,thresholds_rates,False,"The model knowingly used FY2025 parameters: a $1,756 maximum and a $291 standard deduction. FY2026 values are $1,789 and $299. It also skipped monthly rounding of net income and the round-up of the contribution."
+us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"The model applied the $4,250 resource limit for households with a disabled member. NJ categorical eligibility through TANF non-cash benefits removes the asset test, so the $69,500 bank balance is irrelevant and the benefit is $1,259/month."
+us,scenario_008,snap,inkling,llm_error,thresholds_rates,False,"The model used an $1,806 maximum instead of FY2026's $1,789. It also used a $529 contribution instead of rounding 30% of $1,764 up to $530. The result was $1,276/month instead of $1,259."
+us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value and no explanation, so there was no answer to score against the $15,108 reference."
+us,scenario_008,snap,kimi-k3,llm_error,other,False,"The model had the correct $299 deduction, $1,789 maximum, and $1,763.92 net income but skipped SNAP rounding. Net income rounds to $1,764 and the contribution rounds up to $530, giving $1,259/month instead of $1,259.825."
+us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"The model disqualified the household under a $3,750 asset limit for disabled-member households. NJ broad-based categorical eligibility through TANF non-cash benefits has no asset test, so the household receives $1,259/month."
+us,scenario_008,snap,muse-spark-1.3,llm_error,thresholds_rates,False,"The model gave a rounded guess of about $1,300/month instead of computing the $1,789 maximum minus a $530 contribution on $1,764 net income, which equals $1,259/month."
+us,scenario_008,snap,muse-wonder,llm_error,thresholds_rates,False,"The model used a $1,695 maximum allotment for 8 people instead of FY2026's $1,789. Its net income of $1,772 implies a $291 standard deduction instead of $299. The result was $1,163/month instead of $1,259."
+us,scenario_008,snap,ox-alpha,llm_error,thresholds_rates,False,"The model used a $1,795 maximum instead of $1,789 and a $297 standard deduction instead of $299. It then annualized to $15,183, which is not even $1,265 × 12. The correct benefit is $1,789 − $530 = $1,259/month."
+us,scenario_008,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model left out interest income and used a $202 standard deduction and a $1,838 maximum instead of $299 and $1,789. It then made an arbitrary cut 'for housing assistance', even though housing assistance does not reduce the allotment here."
+us,scenario_008,snap,qwen3.8-max,llm_error,thresholds_rates,False,"The model claimed $2,567/month exceeds 130% of poverty for 8 people. The 130% gross limit for 8 is about $6,000/month, and this household's gross income is only 57% of poverty, so it passes and receives $1,259/month."
us,scenario_008,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Declared the spouse ineligible ""under adult NJ rules"" without running New Jersey's ACA expansion income test, which covers adults 19-64 up to 138% FPL and requires no categorical hook such as pregnancy or caretaker status. Household MAGI of about $30,900 against the eight-person FPL is 0.55 x FPL, far inside that limit; the bare denial is consistent with comparing the spouse's own $26,800 of wages to a one- or two-person 138% FPL line (roughly $22,600 to $30,700) instead of household MAGI over a family size of eight."
us,scenario_008,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Defaulted to 0 by treating Medicaid eligibility as something requiring an affirmative listed trigger fact, extending the prompt's ""treat unlisted inputs as false"" rule to the computed output itself. Every input the determination needs was supplied - age 41, non-dependent, New Jersey, household income - and the MAGI adult-expansion test it never ran places household MAGI at 0.55 x FPL against the 138% FPL threshold, which returns eligible."
us,scenario_008,spouse_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"Ran the MAGI test correctly - about $31,420 of household income against the roughly $73,637 138%-FPL line for eight people - then reversed its own correct result with a fabricated rule that receipt of housing assistance disqualifies adults from Medicaid. Housing assistance is excluded from MAGI and disqualifies no one from any Medicaid category, and the MAGI adult category applies no resource test, so neither the housing subsidy nor the $69,500 in bank accounts affects eligibility."
@@ -739,101 +748,103 @@ us,scenario_008,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligib
us,scenario_008,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model inferred breastfeeding or postpartum status and nutritional risk solely from the presence of a one-year-old, despite the prompt requiring unlisted statuses to be false. With neither pregnancy, breastfeeding, nor postpartum status listed, the spouse fails WIC's categorical test regardless of income."
us,scenario_008,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly used disability as a WIC categorical qualifier. Disability and income eligibility do not establish WIC eligibility for an adult who is not pregnant, breastfeeding, or postpartum."
us,scenario_008,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It gave $13,000 in exemptions by allowing separate blind and disabled exemptions, but the spouse gets only one $1,000 blind-or-disabled exemption. It also took the $1,251 renter property-tax deduction (18% of rent), which the reference does not apply. Taxable income is $18,915, not $16,664."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The $12,000 exemption total was correct, but it then subtracted a $1,250.82 renter property-tax deduction. The reference applies no property-tax deduction: at a 1.4% rate the deduction is worth only $17.51, less than the $50 property tax credit, and that credit is refundable and outside this output. Taxable income stays at $18,915, for $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed that the American Opportunity Credit, a pension exclusion and a property tax deduction bring NJ tax to zero. NJ has no AOC, and no pension income exists here. Only $12,000 in exemptions apply, which leaves $18,915 taxed at 1.4%, for $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It asserted that exemptions wipe out NJ tax but did no calculation. The $12,000 in exemptions leaves $18,915 of the $30,915 gross income taxable, and 1.4% of that is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It computed about $251 using $13,000 in exemptions, counting the spouse's blindness and disability as separate $1,000 exemptions instead of one. It then rounded the tax to $0 based on unnamed nonrefundable credits. No NJ nonrefundable credit applies here, so the tax is 1.4% of $18,915, or $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It treated $30,915 of gross income as falling under the $20,000 MFJ filing threshold, but the income is well above it. The $12,000 in exemptions leaves $18,915 taxable at 1.4%."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It used a $1,500 regular exemption for each spouse ($3,000 total), but NJ's regular exemption is $1,000 per spouse, or $2,000 for a joint return. That overstated total exemptions at $13,000 instead of $12,000, so it taxed $17,915 instead of $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It counted the spouse's blindness and disability as two exemptions ($13,000 total) and calculated about $251. It then invented a low-income phase-in that cuts the tax to $0. NJ has no such provision above the $20,000 filing threshold, so 1.4% of $18,915 ($264.81) stands."
-us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied an 'NJ MFJ standard deduction' larger than income. NJ has no standard deduction; it allows only personal exemptions, which total $12,000 here and leave $18,915 taxable."
-us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It started from the federal-style AGI of $30,632, which subtracts half of self-employment tax, but NJ gross income includes the full $4,000 of net self-employment profit, giving $30,915. It also allowed only $9,000 in exemptions, using $1,000 per dependent instead of $1,500. The result was $21,632 taxable instead of $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The $12,000 exemption total was correct, but it subtracted a $181.70 medical expense deduction based on over-the-counter health spending. OTC expenses do not count toward the NJ medical expense deduction, so taxable income stays at $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,age_disability,False,"It took only $11,000 in exemptions and left out the $1,000 blind-or-disabled exemption the spouse qualifies for. That overstated taxable income at $19,915 instead of $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It correctly arrived at $18,915 taxable and $265 of tax, then subtracted the $50 NJ tenant property tax credit. That credit is refundable, so it does not reduce tax before refundable credits."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It asserted that exemptions zero out the tax without any calculation. The $12,000 in exemptions leaves $18,915 taxable at 1.4%, for $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Its preliminary tax of about $251 reflects $13,000 in exemptions, which counts the spouse's blindness and disability twice instead of as one $1,000 exemption. It then subtracted the $50 tenant property tax credit, which is refundable and excluded from this output."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It claimed $14,000 in exemptions, using $3,000 for a joint return instead of $2,000 ($1,000 per spouse) and $2,000 for blind plus disabled instead of one $1,000 exemption. That understated taxable income at $16,915 instead of $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave $0 with no calculation. NJ gross income of $30,915 minus $12,000 in exemptions leaves $18,915, taxed at 1.4% for $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It counted separate $1,000 exemptions for blindness and disability, giving $13,000 instead of $12,000, because NJ allows only one $1,000 blind-or-disabled exemption per person. It also subtracted an 18%-of-rent property-tax deduction of $1,251 that the reference doesn't take. That gave $16,664 of taxable income instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The $12,000 exemption total is right, but it also subtracted a $1,250.82 renter property-tax deduction (18% of $6,949). The reference takes no property-tax deduction, so taxable income should stay at $18,915, and 1.4% of that is $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It started from $30,632, which is federal AGI net of half the self-employment tax; NJ gross income has no such adjustment and is $30,915. It also took a $187 medical deduction for OTC expenses that the reference doesn't allow. Both errors pushed taxable income below $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It applied the federal American Opportunity Credit, a pension exclusion and a property-tax deduction to NJ tax. NJ has no AOC, and none of these benefits zero out the tax. $30,915 minus $12,000 of exemptions leaves $18,915, taxed at 1.4%."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed exemptions eliminate the tax without computing anything. $30,915 of NJ gross income minus $12,000 of exemptions leaves $18,915 of positive taxable income, and 1.4% of that is $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It reached about $251 by counting blindness and disability as two $1,000 exemptions. It then rounded the result to $0 by citing unnamed credits and a below-threshold rule that doesn't apply, since $30,915 exceeds the $20,000 MFJ threshold and no nonrefundable credit offsets the tax."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It treated $30,915 of gross income as below the $20,000 MFJ filing threshold, which it isn't. The household owes tax on $18,915 (gross income less $12,000 of exemptions) at 1.4%."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It gave each spouse a $1,500 personal exemption, for $3,000, when NJ allows $1,000 each, so total exemptions came to $13,000 instead of $12,000. Taxable income became $17,915 instead of $18,915, giving $251 instead of $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It counted separate blind and disabled exemptions ($13,000 total), and then reduced the roughly $251 result to $0 with an invented low-income phase-in. NJ has no such rule above the $20,000 threshold; the correct tax is 1.4% × $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It applied an MFJ standard deduction, but New Jersey has no standard deduction, only exemptions. With $12,000 of exemptions, taxable income is $18,915 and the tax is $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used federal AGI of $30,632, net of half the self-employment tax, instead of NJ gross income of $30,915. It also subtracted only the $9,000 of dependent exemptions, leaving out the $2,000 of regular exemptions and the $1,000 blind/disabled exemption. That pushed taxable income to $21,632 and into the 1.75% bracket."
+us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The $12,000 of exemptions is right, but it subtracted a $181.70 medical deduction built from OTC health expenses. The reference allows no medical deduction, so taxable income is $18,915, not $18,733."
+us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It left out the spouse's $1,000 blind-or-disabled exemption, using $11,000 of exemptions instead of $12,000. Taxable income was overstated at $19,915 instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Taxable income of $18,915 and the $265 tax are right, but it subtracted the $50 NJ property tax credit as if it were nonrefundable. That credit is refundable, so it doesn't reduce tax before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It asserted that exemptions eliminate the tax without computing anything. $30,915 minus $12,000 of exemptions leaves $18,915 taxable at 1.4%, which is $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Its preliminary tax of $251 reflects $13,000 of exemptions (blindness and disability counted separately) instead of $12,000. It then subtracted the $50 property tax credit, which is refundable in NJ and doesn't reduce tax before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used $3,000 of regular MFJ exemptions instead of $2,000, plus $2,000 for blind and disabled instead of the single $1,000 blind-or-disabled exemption. That made total exemptions $14,000 instead of $12,000 and understated taxable income at $16,915."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave $0 with no derivation. The household has $18,915 of NJ taxable income after $12,000 of exemptions, which produces $264.81 at 1.4%."
us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It correctly computed $264.81 on $18,915 of taxable income, then subtracted the $50 NJ property tax credit. That credit is refundable and is excluded from tax before refundable credits."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,age_disability,False,"It claimed separate $1,000 exemptions for the spouse's blindness and disability, but NJ allows one $1,000 blind-or-disabled exemption per person. That brought exemptions to $13,000 and taxable income to $17,915 instead of $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It counted the spouse's blindness and disability twice ($13,000 in exemptions instead of $12,000). It also subtracted a $1,250.82 renter property-tax deduction that the reference does not apply. Taxable income is $18,915, not $16,664."
-us,scenario_008,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,It submitted no value or explanation for state_income_tax_before_refundable_credits.
-us,scenario_008,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It invented a $4,000 MFJ standard deduction, which NJ does not have, and used $1,000 per dependent instead of $1,500. It also subtracted the 18%-of-rent property-tax deduction, which the reference does not apply. The correct base is $30,915 minus $12,000 in exemptions, or $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It assumed exemptions and credits eliminate the tax without computing anything. The $12,000 in exemptions leaves $18,915 taxable, no nonrefundable NJ credit applies, and the tax is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted a $0 tax with no calculation. NJ gross income of $30,915 minus $12,000 in exemptions leaves $18,915 taxed at 1.4%, for $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The $12,000 exemption total was correct, but it subtracted a $181.70 medical deduction based on over-the-counter health expenses. OTC spending does not count for the NJ medical expense deduction, so taxable income is $18,915, not $18,733.30."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It offset the tax with the NJ Child Tax Credit, but that credit is refundable and falls outside tax before refundable credits. Tax on $18,915 at 1.4% is $264.81, with no nonrefundable offset."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It took a medical deduction based on OTC health expenses, which do not qualify. It also subtracted the $50 renter property tax credit as if it were nonrefundable, but that credit is refundable. Neither belongs in this figure, which is 1.4% of $18,915, or $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"It took separate exemptions for blindness and disability, but the spouse gets one $1,000 blind-or-disabled exemption. That inflated total exemptions to $13,000, so it taxed $17,915 instead of $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted an estimated $181.70 medical deduction based on OTC health expenses, which do not count toward the NJ medical expense deduction. Taxable income is $18,915 after the $12,000 in exemptions, not $18,733.30."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It computed the correct $264.81, then subtracted the $50 renter property tax credit. That credit is refundable and does not reduce tax before refundable credits."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It applied an NJ standard deduction, which does not exist; NJ allows only personal and dependent exemptions. Those total $12,000 and leave $18,915 taxable at 1.4%."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It used the federal AGI of $30,632, which subtracts half of SE tax, instead of NJ gross income of $30,915, which includes the full $4,000 business profit. It also used $1,000 per dependent instead of $1,500, giving $9,000 in exemptions instead of $12,000. That pushed taxable income to $21,632 and into the 1.75% bracket."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It reached the correct $18,915 taxable and $265 of tax, then subtracted the $50 renter property tax credit as if it were nonrefundable. NJ's property tax credit is refundable and excluded from this output."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It treated $800 of over-the-counter health expenses as qualifying for the NJ medical expense deduction and subtracted $181.70. OTC expenses do not qualify, so taxable income stays at $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It left out the spouse's $1,000 blind-or-disabled exemption, using $11,000 instead of $12,000. It also applied 1.75% to all of taxable income, but the first $20,000 of joint taxable income is taxed at 1.4%."
-us,scenario_008,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It understated exemptions at $10,000 instead of $12,000; dependents get $1,500 each, which totals $9,000. It also subtracted the $1,251 renter property-tax deduction, which the reference does not apply."
-us,scenario_008,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It submitted no value or explanation for state_income_tax_before_refundable_credits.
-us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It correctly computed $264.81 on $18,915, then subtracted the $50 renter property tax credit. That credit is refundable and is excluded from tax before refundable credits."
-us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It started from the federal AGI of $30,632, which reflects the half-SE-tax adjustment NJ does not allow, instead of NJ gross income of $30,915. It also left out the spouse's $1,000 blind-or-disabled exemption, giving $11,000 in exemptions instead of $12,000."
-us,scenario_008,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,age_disability,False,"It took only $11,000 in exemptions and left out the spouse's $1,000 blind-or-disabled exemption. That overstated taxable income at $19,915 instead of $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,muse-wonder,llm_error,age_disability,False,"It counted separate $1,000 exemptions for the spouse's blindness and disability, but NJ allows a single blind-or-disabled exemption per person. That gave $13,000 in exemptions instead of $12,000, and it taxed $17,915 instead of $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It subtracted a $181.70 medical deduction based on OTC health expenses, which do not qualify for the NJ medical expense deduction. It then subtracted the $50 renter credit, which is refundable and excluded from tax before refundable credits."
-us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It used $1,000 per dependent instead of $1,500, which understated exemptions. It then invented a 'nonrefundable exemption reduction' that brings the tax to $0. The correct $12,000 in exemptions leaves $18,915 taxed at 1.4%, for $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed exemptions and deductions reduce taxable income to zero, but the $12,000 in exemptions leaves $18,915 of the $30,915 gross income taxable. At 1.4% the tax is $264.81."
-us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It phased the federal EITC down to $6,110, even though earned income near $30,500 is below the MFJ phase-out start, so the full $8,231 maximum applies. It then gave $400 per young child by choosing the CTC tier from gross income. NJ taxable income after exemptions is under $30,000, and the 2026 schedule pays $1,250 per child. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Its NJ EITC of $3,292.40 is correct. It left out the $50 NJ renter property tax credit, even though the spouse pays $6,949 in rent. It also used the earlier $1,000-per-child CTC instead of the 2026 schedule's $1,250 per child ($2,500 for the children aged 5 and 1)."
-us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed that New Jersey has no refundable income tax credits. The NJ EITC (40% of federal, $3,292.40), the NJ Child Tax Credit ($2,500) and the $50 property tax credit are all refundable and all apply to this household."
-us,scenario_008,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It reported that no NJ refundable credits apply. This low-income MFJ family with six children gets the 40% NJ EITC ($3,292.40), a $2,500 NJ Child Tax Credit for two children under 6 and the $50 renter property tax credit."
-us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Its reasoning comes to about $2,880 EITC plus $1,000 CTC, yet it submitted an unrelated $9,492. Along the way it phased the federal EITC below the $8,231 maximum, counted only one child under 6 when ages 5 and 1 both qualify, and left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The EITC of $3,292 is correct. It cut the CTC to $600 per child because income was 'near $30,500', but the tier depends on NJ taxable income, which is under $30,000 after exemptions, and the 2026 schedule pays $1,250 per child. Its submitted total also does not match its own components, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It said the NJ Child Tax Credit does not apply at this income. The two children aged 5 and 1 qualify for $2,500 at NJ taxable income under $30,000. It also put the federal EITC at about $7,700 instead of the $8,231 maximum, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"The EITC ($3,292.40) and the $50 property tax credit are both correct. For the two children under 6 it used the earlier $1,000-per-child NJ CTC. The June 30, 2026 schedule for tax years 2026–2028 pays $1,250 each at low taxable income, so the CTC is $2,500, not $2,000."
-us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It started the EITC phase-out at $25,511 and used a $7,830 maximum. That cut the federal EITC to $6,716, but the MFJ phase-out starts above this family's roughly $30,500 earned income, so the full $8,231 applies. It also used $500 per young child instead of $1,250, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It estimated the NJ EITC at about $3,800. That is above the 40% × $8,231 = $3,292.40 ceiling. It also used the earlier $1,000-per-child CTC instead of $1,250, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It placed the household in the $30,001–$40,000 CTC tier ($800 per child) based on gross income. The NJ CTC uses NJ taxable income, which is under $30,000 after the personal and $1,500-per-dependent exemptions, and the 2026 schedule then pays $1,250 per child. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It counted only the NJ EITC and used an $8,021 federal figure instead of the $8,231 2026 maximum. It left out the $2,500 NJ Child Tax Credit for the children aged 5 and 1 and the $50 renter property tax credit."
-us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It counted only the NJ EITC and understated the federal EITC at $7,787 instead of the full $8,231. It left out the $2,500 NJ Child Tax Credit and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It used a federal EITC maximum of $8,190 instead of $8,231. For the two children under 6 it used the earlier $1,000-per-child NJ CTC instead of $1,250 each, and it left out the $50 renter property tax credit."
-us,scenario_008,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It overstated the federal EITC maximum as $8,247 instead of $8,231. It counted a $75 renters' credit, but the NJ property tax credit is $50. It also used $1,000 per young child instead of the 2026 schedule's $1,250."
-us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It put the federal EITC at $7,691 instead of the unreduced $8,231 maximum. It took the $800 CTC tier for income between $30,000 and $40,000 from gross income, when NJ taxable income is under $30,000 and the 2026 schedule pays $1,250 per child. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It found no NJ refundable credits. The household qualifies for the 40% NJ EITC ($3,292.40), the $2,500 NJ Child Tax Credit for two children under 6 and the $50 renter property tax credit."
-us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It applied the $800-per-child CTC tier, which comes from gross income above $30,000. NJ taxable income after exemptions is under $30,000, where the 2026 schedule pays $1,250 per child. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It phased the federal EITC down to $7,128, but earned income is below the MFJ phase-out start, so the full $8,231 applies. It also used the $800 gross-income CTC tier instead of $1,250 per child at NJ taxable income under $30,000, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no derivation and submitted $1,200. The correct components are NJ EITC $3,292.40, NJ CTC $2,500 and property tax credit $50. Its figure leaves out most of the EITC and the child credit."
-us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Its NJ EITC of $3,219.60 is based on an outdated federal maximum of about $8,049 instead of $8,231. Its $1,600 CTC uses the $800 gross-income tier instead of $1,250 per child at NJ taxable income under $30,000. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $4,776 is $1,066.40 short of the correct $3,292.40 EITC + $2,500 CTC + $50 property tax credit. The gap matches an $800-per-child CTC taken from the gross-income tier, an EITC below 40% of the $8,231 maximum, and no property tax credit."
-us,scenario_008,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The $50 property tax credit is correct. Its EITC estimate of $3,293.20 exceeds the reference's $3,292.40 by $0.80. It applied the $800-per-child CTC tier as if NJ income exceeded $30,000. NJ taxable income after exemptions is under $30,000, and the 2026 schedule pays $1,250 per child, for $2,500."
-us,scenario_008,state_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"It overstated the federal EITC at about $8,268 instead of $8,231. It used the earlier $1,000-per-child CTC instead of $1,250, and it left out the $50 renter property tax credit."
-us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It gave no answer for state_refundable_credits, so there is no value to score against the $5,842.40 total."
-us,scenario_008,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It counted only one child under 6, but children aged 5 and 1 both qualify for the NJ CTC. It used $500 per child instead of $1,250 and overstated the federal EITC at $8,360 instead of $8,231. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It concluded that no NJ refundable credit applies. The family's earned income qualifies it for the 40% NJ EITC ($3,292.40), the two children under 6 qualify for the $2,500 NJ CTC, and the renters get the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It returned zero on the grounds that no special circumstances were given. Low earned income and young children are exactly what the NJ EITC ($3,292.40) and NJ CTC ($2,500) require, and the renters also get the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The EITC is correct. It left out the $50 NJ renter property tax credit, even though the spouse pays $6,949 in rent. It also used the earlier $1,000-per-child CTC instead of the 2026 schedule's $1,250 per child."
-us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It counted only the NJ EITC and used the outdated $8,046 federal maximum instead of $8,231. It left out the $2,500 NJ Child Tax Credit for the children aged 5 and 1 and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $1,600 CTC applies the $800 tier for income over $30,000. NJ taxable income after exemptions is under $30,000, where the 2026 schedule pays $1,250 per child. It also left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its total of $4,892.40 means a $1,600 CTC at the $800 gross-income tier and no property tax credit. The correct CTC is $2,500: $1,250 per child at NJ taxable income under $30,000. The $50 renter credit is also missing."
-us,scenario_008,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The EITC and the $50 property tax credit are correct. It used the $800-per-child CTC tier as though NJ income exceeded $30,000. NJ taxable income after exemptions is under $30,000, and the 2026 schedule pays $1,250 per child."
-us,scenario_008,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It computed the NJ EITC at 50% of the federal credit, but the NJ rate is 40%, which gives $3,292.40. It also used $1,000 per young child instead of $1,250 and left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It overstated the NJ EITC at $3,337 instead of $3,292.40. It used the $800 gross-income CTC tier instead of $1,250 per child at NJ taxable income under $30,000, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The EITC ($3,292.40) and the $50 property tax credit are correct. It used the earlier $1,000-per-child NJ CTC instead of the June 30, 2026 schedule, which pays $1,250 for each of the two children under 6, so it is $500 short."
-us,scenario_008,state_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,"Take out the $50 property tax credit and its total leaves $4,942.80 for EITC and CTC, against the correct $5,792.40. It applied an earlier CTC schedule paying at most $1,000 per child instead of $1,250, and its EITC differs from 40% × $8,231 = $3,292.40."
-us,scenario_008,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It said no state refundable credits apply. The household qualifies for the NJ EITC ($3,292.40), the NJ CTC ($2,500) and the $50 renter property tax credit."
-us,scenario_008,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It chose the CTC tier from gross income ($30,001–$40,000) and misremembered even that tier as $400 per child. NJ taxable income after exemptions is under $30,000, where the 2026 schedule pays $1,250 per child. It also overstated the federal EITC as $8,247 and left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It used $500 per child under 6 for the NJ CTC instead of the 2026 schedule's $1,250. It put the federal EITC at $8,250 instead of $8,231, and it said no other refundable NJ credits apply, leaving out the $50 renter property tax credit."
-us,scenario_008,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It explicitly refused the $50 property tax credit for lack of rent, but the spouse's $6,949 pre-subsidy rent is listed, so the renter credit applies. It also used the earlier $1,000-per-child CTC instead of $1,250."
-us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It counted only the NJ EITC and understated the federal EITC at $7,901 instead of $8,231. It stated that no other credits apply, leaving out the $2,500 NJ CTC for the two children under 6 and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,inkling,llm_error,state_local_rule,False,"It counted only the NJ EITC, with an overstated $8,271 federal base. It left out the $2,500 NJ Child Tax Credit for the children aged 5 and 1 and the $50 renter property tax credit."
-us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no answer for state_refundable_credits, so there is no value to score against the $5,842.40 total."
-us,scenario_008,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The EITC is correct. It left out the $50 NJ renter property tax credit, even though the spouse pays rent. It also used the earlier $1,000-per-child CTC instead of the 2026 schedule's $1,250 per child."
-us,scenario_008,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It said no NJ refundable credits apply. The household qualifies for the 40% NJ EITC ($3,292.40), the $2,500 NJ CTC and the $50 property tax credit."
-us,scenario_008,state_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"It understated the federal EITC at about $7,800, which gives a $3,120 NJ EITC instead of $3,292.40. It used $500 per young child instead of $1,250, and it left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"It used the outdated $8,046 federal EITC maximum instead of $8,231. It used the earlier $1,000-per-child CTC instead of $1,250, and it left out the $50 renter property tax credit."
-us,scenario_008,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It phased the CTC down because gross income of $30,915 is over $30,000. The NJ CTC tier uses NJ taxable income, which is under $30,000 after exemptions, and pays $1,250 per child under the 2026 schedule. It also overstated the federal EITC at $8,262 and left out the $50 property tax credit."
-us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It claimed the 3+-child federal EITC phases out to about $0 at $30,800 of earned income. That income is below the MFJ phase-out start, so the full $8,231 applies and the NJ EITC is $3,292.40. It also dropped the $2,500 NJ CTC and the $50 property tax credit, with no rule supporting a $0 total."
-us,scenario_008,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It invented an NJ EITC phase-out that removes the credit and a $100 'minimum refundable credit'. The NJ EITC is simply 40% of the federal EITC, which is the full $8,231 here, giving $3,292.40. The $2,500 CTC and the $50 property tax credit also apply."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It claimed separate $1,000 exemptions for blindness and disability. NJ allows a single $1,000 blind-or-disabled exemption here, so exemptions were $13,000 instead of $12,000 and taxable income was $17,915 instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It double-counted blind and disabled exemptions ($13,000 instead of $12,000) and subtracted a $1,250.82 renter property-tax deduction that the reference doesn't take. That understated taxable income at $16,664 instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It gave no value or reasoning for state_income_tax_before_refundable_credits, so the required output is missing."
+us,scenario_008,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It valued the six dependents at $1,000 each instead of $1,500 and added a $4,000 MFJ standard deduction that NJ doesn't have. It also subtracted a $1,251 renter property-tax deduction. Taxable income should be $30,915 minus only $12,000 of exemptions, or $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It assumed exemptions and credits eliminate the tax. Exemptions total only $12,000, leaving $18,915 taxable, and no nonrefundable NJ credit offsets the $264.81 tax."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed the tax base after deductions is too low to owe tax. NJ taxable income is $18,915 after $12,000 of exemptions, which produces $264.81 at 1.4%."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The $12,000 of exemptions is right, but it subtracted a $181.70 medical deduction from OTC health expenses. The reference allows no medical deduction, so taxable income is $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It offset the tax with the NJ child tax credit, which is refundable and therefore excluded from tax before refundable credits. The pre-refundable liability is 1.4% × $18,915 = $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It took a medical deduction from OTC expenses that the reference doesn't allow. It also subtracted the $50 renter property tax credit, which is refundable and doesn't reduce tax before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $250.81 answer corresponds to $17,915 of taxable income, meaning it gave the spouse separate blindness and disability exemptions. NJ allows one $1,000 blind-or-disabled exemption, so exemptions are $12,000 and taxable income is $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted an estimated $181.70 medical deduction for OTC expenses. The reference allows no medical deduction, so taxable income stays at $18,915 instead of $18,733."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It correctly computed $264.81 on $18,915 of taxable income, then subtracted the $50 renter property tax credit. That credit is refundable in NJ and is excluded from tax before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It invoked an NJ standard deduction, which doesn't exist. Exemptions of $12,000 leave $18,915 taxable, which produces $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used federal AGI of $30,632 instead of NJ gross income of $30,915, and valued dependents at $1,000 instead of $1,500 each, for $9,000 of exemptions instead of $12,000. That pushed taxable income to $21,632 and into the 1.75% bracket."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It correctly found $18,915 of taxable income and $265 of tax, then subtracted the $50 renter property tax credit. That credit is refundable and doesn't reduce tax before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The $12,000 of exemptions is right, but it subtracted a $181.70 medical deduction from OTC expenses above 2% of income. The reference allows no such deduction, so taxable income is $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It left out the $1,000 blind-or-disabled exemption ($11,000 instead of $12,000). It also taxed all of its $19,915 at 1.75%, even though MFJ income up to $20,000 is taxed at 1.4%."
+us,scenario_008,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used only $10,000 of exemptions instead of the correct $12,000, which is $2,000 regular, $9,000 for dependents and $1,000 blind/disabled. It also subtracted a $1,251 renter property-tax deduction that the reference doesn't take."
+us,scenario_008,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or reasoning for state_income_tax_before_refundable_credits, so the required output is missing."
+us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It correctly computed $264.81 on $18,915, then subtracted the $50 renter property tax credit. That credit is refundable in NJ and is excluded from tax before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It started from federal AGI of $30,632 (net of half the self-employment tax) instead of NJ gross income of $30,915, which has no such adjustment. It also left out the spouse's $1,000 blind-or-disabled exemption, using $11,000 instead of $12,000."
+us,scenario_008,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It used $11,000 of exemptions and left out the spouse's $1,000 blind-or-disabled exemption, which overstated taxable income at $19,915 instead of $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It gave separate exemptions for blindness and disability, for $13,000 in total. NJ allows a single $1,000 blind-or-disabled exemption, so exemptions are $12,000 and taxable income is $18,915."
+us,scenario_008,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It subtracted a $181.70 OTC medical deduction that the reference doesn't allow. It also subtracted the $50 NJ renter property tax credit, which is refundable and excluded from tax before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It valued dependents at $1,000 instead of $1,500. It then reduced the tax to $0 with an invented nonrefundable exemption reduction. The correct derivation is $30,915 minus $12,000 of exemptions, or $18,915 taxed at 1.4%."
+us,scenario_008,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed NJ exemptions reduce taxable income to zero. They total only $12,000 against $30,915 of gross income, leaving $18,915 taxable and $264.81 of tax."
+us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Phased the federal EITC down to about $6,110, even though earned income of about $30,500 is below the 2026 joint-filer phaseout start, so the full $8,231 applies and the NJ EITC is $3,292.40. It also gave the NJ CTC at $400 per child based on AGI instead of NJ taxable income (under $30,000), left out the $50 renter property tax credit, and missed the June 2026 CTC schedule that pays $2,500 total."
+us,scenario_008,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Got the NJ EITC right at $3,292.40 but left out the $50 NJ property tax credit, which the household gets because it pays $6,949 in rent. It also used the old $1,000-per-child CTC ($2,000) instead of the June 30, 2026 schedule, which pays $2,500 for the two children under 6."
+us,scenario_008,state_refundable_credits,claude-fable-5.2,llm_error,state_local_rule,False,"Got the $3,292.40 NJ EITC and the $50 property tax credit right, but used the old NJ CTC of $1,000 per child under 6 ($2,000). The schedule New Jersey enacted on June 30, 2026 for tax years 2026–2028 pays $2,500 for these two young children at this low taxable income."
+us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly said New Jersey has no refundable income tax credits. NJ has a refundable EITC (40% of federal, $3,292.40 here), a refundable child tax credit ($2,500 for the two children under 6), and a $50 property tax credit for renters."
+us,scenario_008,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Said no NJ refundable credits apply, even though this low-income joint filer with six children gets the refundable NJ EITC (40% × $8,231 = $3,292.40), the NJ child tax credit ($2,500 for the children aged 5 and 1), and the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Its own components (about $2,880 EITC plus $1,000 CTC) add up to about $3,880, yet it submitted $9,492, which matches no calculation. It also phased the federal EITC below the $8,231 maximum, counted only the age-1 child for the NJ CTC when the age-5 child also qualifies, and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Got the NJ EITC right at $3,292 but put the NJ CTC at the $600 tier using AGI near $30,500. The tier depends on NJ taxable income, which is well under $30,000 after exemptions, and the 2026 schedule pays $2,500. It then added about $892 of unexplained 'minor amounts' instead of the $50 property tax credit."
+us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Said the NJ child tax credit does not apply at this income, but NJ taxable income is well under $30,000, so the two children under 6 qualify for $2,500. It also understated the federal EITC as about $7,700 instead of the $8,231 maximum and left out the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Got the $3,292.40 EITC and the $50 property tax credit right, but used the old NJ CTC of $1,000 per child under 6 ($2,000). The June 30, 2026 schedule for tax years 2026–2028 pays $2,500 for these two children."
+us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Used an outdated $7,830 maximum and phased the federal EITC from a $25,511 threshold. A joint filer with about $30,500 of earned income is below the 2026 joint phaseout start, so the full $8,231 applies (NJ EITC $3,292.40). It also used $500 per child for the NJ CTC instead of $2,500 total and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Guessed a federal EITC of $8,000–$9,000 and an NJ EITC of about $3,800, which is above the cap: 40% of the $8,231 maximum is $3,292.40. It also used the old $2,000 CTC instead of the 2026 schedule's $2,500 and left out the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Put the NJ CTC in the $30,001–$40,000 bracket ($800 per child) using gross income. The bracket depends on NJ taxable income, which is well under $30,000 after exemptions for eight people, so the 2026 schedule pays $2,500. It also left out the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Counted only the NJ EITC, and on a $8,021 federal figure instead of the $8,231 maximum. It left out the refundable NJ child tax credit ($2,500 for the children aged 5 and 1) and the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Counted only the NJ EITC, on a $7,787 federal EITC instead of the full $8,231 for 3+ children below the joint phaseout. It left out the $2,500 NJ child tax credit for the two children under 6 and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Used a $8,190 federal EITC instead of the $8,231 maximum, used the old $2,000 NJ CTC instead of the 2026 schedule's $2,500, and left out the $50 property tax credit for this renting household."
+us,scenario_008,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Overstated the federal EITC as $8,247 instead of $8,231, gave the renter property tax credit as $75 instead of $50, and used the old $2,000 NJ CTC instead of the $2,500 that the June 2026 schedule pays for the two children under 6."
+us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Phased the federal EITC down to $7,691 when the full $8,231 applies, and used the $800 tier for $30,000–$40,000 income based on gross income instead of NJ taxable income (under $30,000), where the 2026 schedule pays $2,500. It also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Found no NJ refundable credits, missing the 40% NJ EITC ($3,292.40), the NJ child tax credit for the two children under 6 ($2,500), and the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Used the $800-per-child CTC tier, putting income above $30,000 on a gross-income basis. NJ taxable income after exemptions is well under $30,000, and the 2026 schedule pays $2,500 for the two young children. It also left out the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Phased the federal EITC down to $7,128, even though earned income is below the 2026 joint phaseout start, so the full $8,231 applies (NJ EITC $3,292.40). It also used the $800 CTC tier instead of the under-$30,000 taxable-income tier ($2,500 under the 2026 schedule) and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Gave no breakdown. The correct sum is the $3,292.40 NJ EITC (40% of the $8,231 maximum), the $2,500 CTC for two children under 6, and the $50 property tax credit; $1,200 roughly matches a partial CTC alone, with no NJ EITC at all."
+us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Understated the NJ EITC ($3,219.60 instead of 40% × $8,231 = $3,292.40), used the $800-per-child CTC tier instead of the under-$30,000 taxable-income tier that pays $2,500 under the 2026 schedule, and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Its $4,776 matches a phased-down NJ EITC of about $3,176 plus the $1,600 CTC tier for $30,000–$40,000 income. The correct figures are the full $3,292.40 EITC, the $2,500 CTC under the 2026 schedule (based on NJ taxable income under $30,000), and the $50 property tax credit, which it left out."
+us,scenario_008,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Got the EITC and the $50 property tax credit right, but put the NJ CTC in the $800-per-child bracket using gross income. NJ taxable income after exemptions is well under $30,000, and the June 2026 schedule pays $2,500 for the two children under 6."
+us,scenario_008,state_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"Overstated the federal EITC (implied $8,268 instead of $8,231), used the old $2,000 CTC instead of the 2026 schedule's $2,500, and left out the $50 property tax credit for this renting household."
+us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Gave no answer for state_refundable_credits. The correct value is $3,292.40 NJ EITC + $2,500 NJ CTC + $50 property tax credit."
+us,scenario_008,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Used an $8,360 federal EITC instead of $8,231 and counted only one child under 6 at $500, leaving out the age-5 child. The 2026 NJ CTC pays $2,500 for both children aged 5 and 1. It also left out the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Concluded no NJ refundable credit applies, missing the refundable NJ EITC of 40% of the $8,231 federal EITC, the $2,500 NJ child tax credit for the two children under 6, and the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Returned zero by requiring special circumstances. The NJ EITC, NJ child tax credit, and $50 property tax credit only require low earned income, young children, and paid rent, which this household has; together they total $5,842.40."
+us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Got the $3,292.40 NJ EITC right but left out the $50 NJ property tax credit for the household's $6,949 rent. It also used the old $1,000-per-child CTC instead of the June 2026 schedule's $2,500 for the two children under 6."
+us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Counted only the NJ EITC, on an understated $8,046 federal EITC instead of $8,231, and left out the refundable NJ child tax credit ($2,500) and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Got the EITC right but used the $800-per-child CTC tier, treating income as above $30,000. The tier depends on NJ taxable income, which is well under $30,000 after dependent exemptions, so the 2026 schedule pays $2,500. It also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $4,892.40 is $3,292.40 EITC plus a $1,600 CTC at the $800 tier based on gross income. NJ taxable income is under $30,000, so the 2026 schedule pays $2,500, and it also left out the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Got the EITC and the $50 property tax credit right, but put the NJ CTC in the $800-per-child bracket instead of the lowest taxable-income tier. NJ taxable income is under $30,000, and the June 2026 schedule pays $2,500 for the two young children."
+us,scenario_008,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Used a 50% NJ EITC rate; the NJ EITC is 40% of the federal credit, which gives $3,292.40, not $4,081. It also used the old $2,000 CTC instead of the 2026 schedule's $2,500 and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Overstated the NJ EITC at $3,337 (40% of $8,231 is $3,292.40) and used the $800-per-child CTC tier instead of the under-$30,000 taxable-income tier, which pays $2,500 under the 2026 schedule. It also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Got the $3,292.40 EITC and the $50 property tax credit right, but used the old NJ CTC of $1,000 per child under 6 ($2,000). The June 30, 2026 schedule for tax years 2026–2028 pays $2,500 for the two children aged 5 and 1."
+us,scenario_008,state_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,"Its $4,992.80 works out to an overstated $3,342.80 EITC, the $50 property tax credit, and a $1,600 CTC at the $800-per-child tier. The full EITC is $3,292.40, and the 2026 schedule pays $2,500 to these two young children at NJ taxable income under $30,000."
+us,scenario_008,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Said no state refundable credits apply, missing the 40% NJ EITC ($3,292.40), the NJ child tax credit ($2,500 for two children under 6), and the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Used the $30,001–$40,000 gross-income bracket at $400 per child for the NJ CTC. NJ taxable income is under $30,000, and the 2026 schedule pays $2,500 for the two children under 6. It also overstated the federal EITC as $8,247 and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Used $500 per child under 6 for the NJ CTC instead of the lowest-tier 2026 amount ($2,500 total), rounded the federal EITC up to $8,250, and said no other credits apply, which drops the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Wrongly turned down the $50 property tax credit by saying no rent was listed, even though the spouse pays $6,949 in pre-subsidy rent. It also used the old $1,000-per-child CTC instead of the June 2026 schedule's $2,500 for the two children under 6."
+us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Counted only the NJ EITC, on a $7,901 federal figure instead of the $8,231 maximum, and wrongly said no other refundable state credits exist. That leaves out the $2,500 NJ child tax credit and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Counted only the NJ EITC (using an overstated $8,271 federal EITC) and left out the refundable NJ child tax credit ($2,500 for the children aged 5 and 1) and the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no answer for state_refundable_credits. The correct value is $3,292.40 NJ EITC + $2,500 NJ CTC + $50 property tax credit."
+us,scenario_008,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Got the $3,292.40 EITC right but left out the $50 property tax credit for the household's rent, and used the old $1,000-per-child CTC instead of the June 2026 schedule's $2,500 for the two children under 6."
+us,scenario_008,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Said no NJ refundable credits apply, missing the 40% NJ EITC on the $8,231 federal credit, the NJ child tax credit for the two children under 6, and the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"Understated the NJ EITC at $3,120 instead of $3,292.40 and put the NJ CTC at $500 per child ($1,000) instead of the lowest-income-tier 2026 amount of $2,500. It also left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Used an $8,046 federal EITC instead of the $8,231 maximum, used the old $2,000 CTC instead of the 2026 schedule's $2,500, and left out the $50 renter property tax credit."
+us,scenario_008,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Phased down the NJ CTC because gross income was $915 over $30,000. The CTC tier depends on NJ taxable income, which is well under $30,000 after exemptions, and the 2026 schedule pays $2,500. It also overstated the federal EITC at $8,262 and left out the $50 property tax credit."
+us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Claimed the federal EITC phases out to near zero at $30,800 for a joint filer with 3+ children. That income is below the joint phaseout start, so the full $8,231 applies and the NJ EITC is $3,292.40. It also wrongly zeroed the $2,500 NJ CTC and the $50 property tax credit."
+us,scenario_008,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Wrongly claimed the household is above an NJ EITC phaseout and made up a $100 minimum credit. The NJ EITC is simply 40% of the full $8,231 federal EITC ($3,292.40), plus the $2,500 NJ CTC and the $50 property tax credit."
us,scenario_008,tanf,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model submitted a bare number with no derivation; the correct derivation applies New Jersey's WFNJ eligibility standard and yields $0, because countable income of about $2,576/month exceeds the family-of-eight standard (roughly $770/month) even after the 50% earned-income disregard, and $69,500 in bank assets exceeds the $2,000 TANF resource limit. Its $5,800 figure (about $483/month) is consistent with pulling a large-family NJ grant-schedule amount and shading it down for earnings — treating TANF as a residual gap-filler for a household with six children — while never running the income eligibility gate or the asset test at all."
us,scenario_009,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Its reasoning reached exactly $8,348: AGI $105,900, minus the $32,200 standard deduction, gives $73,700 taxed at 10% up to $24,800 and 12% above. It then submitted $3,455 anyway, a value with no basis in its own work."
us,scenario_009,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It included the full $30,000 of Social Security instead of the 85% cap of $25,500. It also subtracted the spouse's $8,139 ESI premiums from wages and used a $30,000 standard deduction instead of $32,200. Its final $2,950 doesn't even match its own estimate of about $8,600 in tax."
@@ -988,56 +999,57 @@ us,scenario_012,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output
us,scenario_012,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_012,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the reduced-price ceiling as sufficient for reduced-price eligibility without first assigning the household to the free-meals tier. The household's 0.81 FPG ratio and categorical eligibility produce FREE status, which makes reduced-price eligibility false."
us,scenario_012,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly stated that income below 130% FPG places the child in the free-meal range and that reduced-price support applies only from 130% through 185%, but then submitted 1 contrary to its own derivation. FREE-tier classification, independently reinforced by categorical eligibility, makes the reduced-price output 0."
-us,scenario_012,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It added a roughly $400 SUA because of the energy subsidy and used a $1,023 maximum allotment, which is wrong for 3 people. Its final $378/month is actually the household's 30% expected contribution, not the allotment. The allotment is $785 − $378 = $407 with no shelter deduction."
-us,scenario_012,snap,claude-fable-5.1,llm_error,other,False,"The method was right, but it skipped SNAP's rounding steps. Net income rounds to $1,258, and the 30% contribution (377.4) rounds up to $378. That gives a whole-dollar $407/month allotment, not the unrounded $407.70 annualized to $4,892."
-us,scenario_012,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left out the 20% earned income deduction ($366.67) and used a wrong $256 standard deduction and a $694 maximum allotment. It also counted the energy subsidy as reducing net income. The correct figures are a $209 standard deduction, a $785 maximum, net income of $1,258 and $407/month."
-us,scenario_012,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"It wrongly concluded the household fails the income tests and treated it as 4 people. Gross income of $1,833/month is well under 130% of the $2,220.83 three-person poverty guideline. Net income of $1,258 is under 100%, so the household gets $407/month."
-us,scenario_012,snap,claude-opus-4.7,llm_error,other,False,"It correctly found no excess shelter deduction and a benefit of about $412/month. It then dropped that result for an unsupported $601/month. It also used a $224 standard deduction instead of $209, which would have given $407/month after rounding."
-us,scenario_012,snap,claude-opus-4.8,llm_error,other,False,"It calculated about $406/month with no shelter deduction, then 're-estimated with shelter' to an unsupported $514/month, even though it had already found rent below half of net income. It also used a $204 standard deduction instead of the FY2026 $209."
-us,scenario_012,snap,claude-opus-5,llm_error,other,False,"Its $378/month equals the household's 30% expected contribution (30% of $1,258, rounded up). It reported that contribution as the benefit instead of subtracting it from the $785 maximum to get $407."
-us,scenario_012,snap,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It gave the household a $395 SUA because of the energy subsidy and computed a $216 excess shelter deduction. The engine does not read the energy subsidy when it sets the utility allowance, and the household reports no heating, cooling or other utility expense, so it gets no SUA. Shelter costs are therefore just the $450 rent, which is below half of $1,258, so there is no deduction."
-us,scenario_012,snap,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It counted the unborn child as a fourth SNAP member, using a $973 maximum, and added a $414 SUA because of the energy subsidy. SNAP counts only the three people who exist, and this household gets no utility allowance. Its submitted $5,136 does not even match its own $7,968 calculation."
-us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It treated the pregnancy as adding a fourth household member, using a $994 maximum, and added a utility allowance tied to the energy subsidy. SNAP uses the 3-person $785 maximum with no shelter deduction here."
-us,scenario_012,snap,claude-sonnet-5.5,llm_error,other,False,"The deductions were right, but it estimated the 30% contribution as about $377 instead of rounding it up to $378. It then approximated $408/month as $4,890 a year. The correct whole-dollar allotment is $407/month, or $4,884."
-us,scenario_012,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,It used a $211 standard deduction and a $786 maximum allotment instead of the FY2026 3-person figures of $209 and $785. That produced $409/month instead of $407.
-us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It added a $500 SUA to the rent and computed a $316.67 excess shelter deduction, and it used a $200 standard deduction. This household gets no utility allowance, and the $450 rent alone is below half of net income. Net income is therefore $1,258, not $950."
-us,scenario_012,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used FY2025 parameters: a $768 maximum allotment and a $204 standard deduction. FY2026 uses $785 and $209, which give net income of $1,258, a $378 contribution and $407/month."
-us,scenario_012,snap,deepseek-v4.1-flash,llm_error,household_unit_or_filing_status,False,"It counted the unborn child and set the SNAP household size at 4, which applies the larger 4-person maximum allotment. SNAP counts only the three people who exist, with a $785 maximum, giving $407/month."
-us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It treated the energy subsidy as triggering a standard utility allowance and computed an excess shelter deduction. The engine does not read the subsidy when it sets the utility allowance, and the household reports no utility expense, so it gets no SUA. The $450 rent is below half of the $1,258 net income, so the benefit stays at $407/month."
-us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $446/month implies about $130 of extra monthly deductions beyond the standard and earned income deductions, which matches an excess shelter or utility deduction. No such deduction applies, so net income is $1,258 and the benefit is $407/month."
-us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It applied an excess shelter deduction based on rent plus a standard utility allowance. The household has no SUA entitlement, and the $450 rent alone does not exceed half of the $1,258 net income."
-us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied a shelter deduction that does not exist here. The $450 rent is below half of the $1,258 net income, and no utility allowance applies, so the benefit is $407/month, not $457."
-us,scenario_012,snap,gemini-3.5-flash-lite,llm_error,other,False,"Its roughly $121/month implies countable income far above the $1,258 net income left after the $209 standard deduction and $366.67 earned income deduction. The correct calculation is $785 − $378 = $407/month."
-us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It included an excess shelter deduction, which raised the benefit to $453/month. With the $450 rent below half of the $1,258 net income and no utility allowance, the deduction is zero and the benefit is $407/month."
-us,scenario_012,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It included shelter costs as a deduction. The rent alone falls below the half-of-net-income threshold, and no utility allowance applies, so net income stays at $1,258 and the benefit at $407/month."
-us,scenario_012,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $200/month standard deduction and a $768 maximum allotment ($9,216 a year). The FY2026 figures are $209 and $785, which give $407/month."
-us,scenario_012,snap,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It treated LIHEAP-type energy assistance as conferring the standard utility allowance. The engine does not read the energy assistance when it sets the utility allowance, and the household reports no utility expense, so it gets no SUA and no shelter deduction."
-us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"It added an extra household member for the pregnancy, using 4-person figures ($975 maximum, $238 standard deduction), and added a $360 SUA because of the energy subsidy. The correct unit is 3 people with a $785 maximum, a $209 standard deduction and no shelter deduction."
-us,scenario_012,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"It added a utility allowance because of the $411 energy subsidy, which created an excess shelter deduction. It also used a $204 standard deduction and a $780 maximum. The correct figures are no utility allowance, $209 and $785, giving $407/month."
-us,scenario_012,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $497/month comes from counting housing and utility costs as a deduction. No utility allowance applies, and the $450 rent is below half of the $1,258 net income, so the benefit is $407/month."
-us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned 0 on the grounds that SNAP eligibility flags were missing. The household of 3 meets the work, immigration, asset and income tests, and qualifies for $407/month."
-us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It treated the energy subsidy as qualifying the household for the heating/cooling utility allowance. The engine does not read the subsidy when it sets the utility allowance, and the household reports no utility expense, so there is no excess shelter deduction and net income is $1,258, not about $1,000."
-us,scenario_012,snap,gpt-5.6-luna,llm_error,other,False,"The deductions were right, but it rounded $407.70 up to $408. SNAP rounds net income to $1,258 and rounds the 30% contribution up to $378, which leaves $407/month."
-us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied a heating/cooling utility deduction because of the energy subsidy. That subsidy confers no SUA for this household, and the rent alone produces no excess shelter deduction."
-us,scenario_012,snap,gpt-5.6-terra,llm_error,other,False,"It got $408/month by rounding the unrounded $407.70 to the nearest dollar. The correct steps are to round net income to $1,258 and round the 30% contribution up to $378, giving $407."
-us,scenario_012,snap,gpt-6-astra,llm_error,other,False,"The deductions and parameters were correct, but it worked with unrounded annual figures. It needed to round monthly net income to $1,258 and round the 30% contribution up to $378, which gives a $407/month allotment."
-us,scenario_012,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $406.20 matches using a $204 standard deduction (net income of $1,262.67, contribution of $378.80) instead of the FY2026 $209. It also skipped the rounding of net income and the expected contribution."
-us,scenario_012,snap,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It included a shelter deduction, which pushed the benefit to $504/month. With $450 rent below half of the $1,258 net income and no utility allowance, the shelter deduction is zero."
-us,scenario_012,snap,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It added an estimated $433 heating utility allowance because of the energy subsidy. That subsidy does not confer an SUA for this household, so the only shelter cost is the $450 rent and no excess shelter deduction applies."
-us,scenario_012,snap,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It added an estimated utility allowance on top of rent, creating an excess shelter deduction. No utility allowance applies, so net income is $1,258 and the benefit is $407/month."
-us,scenario_012,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its $300/month matches leaving out the 20% earned income deduction ($785 − 30% of about $1,624). Subtracting the $366.67 earned income deduction brings net income to $1,258 and the benefit to $407/month."
-us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"It used outdated parameters: a $198 standard deduction and a $766 maximum allotment. The FY2026 figures of $209 and $785 give net income of $1,258 and $407/month."
-us,scenario_012,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"It applied a LIHEAP-triggered SUA in an excess shelter deduction. The engine does not read the energy assistance when it sets the utility allowance, and the household reports no utility expense, so there is no shelter deduction."
-us,scenario_012,snap,grok-4.7,llm_error,other,False,"It computed the benefit on unrounded annual totals. SNAP rounds monthly net income to $1,258 and rounds the 30% contribution up to $378, giving a whole-dollar $407/month allotment."
-us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"It subtracted the $210 standard deduction once from annual income instead of 12 times (about $2,508 a year). It also added the $411 energy subsidy to shelter costs, which changed nothing because its $5,811 stayed below half of its income. The single standard deduction inflated countable income and cut the benefit."
-us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"No SNAP value or explanation was given, so nothing could be parsed."
-us,scenario_012,snap,kimi-k3,llm_error,other,False,"The deductions were right, but it used unrounded net income ($1,257.67) and an unrounded 30% contribution. Rounding net income to $1,258 and rounding the contribution up to $378 gives $407/month, not $407.70."
-us,scenario_012,snap,minimax-m3,llm_error,missing_output,False,"It submitted 0 without doing the calculation. The household passes every test, and $785 − $378 gives a $407/month benefit."
-us,scenario_012,snap,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It applied an SUA because of the $411 energy subsidy. The engine does not read the subsidy when it sets the utility allowance, and the household reports no utility expense, so there is no shelter deduction and the benefit is $407/month."
-us,scenario_012,snap,muse-wonder,llm_error,thresholds_rates,False,"It used the FY2025 3-person maximum allotment of $768 instead of the FY2026 $785, which gave $391/month instead of $407."
-us,scenario_012,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"It added a LIHEAP-funded SUA of about $470 to rent to create an excess shelter deduction, and it used a $204 standard deduction. Neither applies: there is no utility allowance for this household, and the FY2026 standard deduction is $209."
-us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the $411 energy subsidy as unearned income, although energy assistance is excluded from SNAP income. It also used outdated parameters ($768 maximum, $198 standard deduction), and its final $5,111 does not follow from its own arithmetic."
-us,scenario_012,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It treated gross earnings of $22,000 as net income and never subtracted the $209 standard deduction or the 20% earned income deduction. Net income is actually $1,258/month, under the $2,220.83 poverty guideline."
+us,scenario_012,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It wrongly gave a standard utility allowance because of the energy subsidy, then abandoned its own math. Its $4,536 equals 12 × $378, which is the household's 30% expected contribution, not the benefit. The benefit is $785 − $378 = $407 a month."
+us,scenario_012,snap,claude-fable-5.1,llm_error,other,False,"The deductions and parameters were right, but it skipped SNAP rounding. Net income rounds to $1,258, and 30% of that ($377.40) rounds up to $378, so the benefit is $407 a month, not $407.70 a month ($4,892 a year)."
+us,scenario_012,snap,claude-fable-5.2,llm_error,other,False,"It took 30% of the unrounded $1,257.67 instead of rounding net income to $1,258 and rounding the $377.40 contribution up to $378. That gave $407.70 a month instead of $407, so $4,892.40 instead of $4,884."
+us,scenario_012,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left out the 20% earned income deduction ($366.67), used an outdated $256 standard deduction and $694 maximum allotment instead of $209 and $785, and treated the energy subsidy as income. That pushed net income far too high and the benefit far too low."
+us,scenario_012,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"It counted 4 people and said the household failed the income tests. In fact, $1,833 gross a month is well under 130% of poverty for 3, and $1,258 net is under the $2,220.83 net limit, so the household qualifies for $407 a month."
+us,scenario_012,snap,claude-opus-4.7,llm_error,other,False,"It correctly found no excess shelter deduction and about $412 a month, using a $224 standard deduction instead of $209. It then switched without support to $601 a month. The correct figure is $785 − $378 = $407 a month."
+us,scenario_012,snap,claude-opus-4.8,llm_error,other,False,"It reached about $406 a month ($4,872 a year) with no shelter deduction, using a $204 standard deduction instead of $209. It then inflated this to $514 a month without justification. The correct benefit is $407 a month."
+us,scenario_012,snap,claude-opus-5,llm_error,other,False,"It gave $378 a month as the benefit, but $378 is the household's 30% expected contribution (30% of $1,258, rounded up). The benefit is $785 − $378 = $407 a month."
+us,scenario_012,snap,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It gave a $395 standard utility allowance because of the $411 energy subsidy and so created a $216 excess shelter deduction. Under 2026 rules, energy assistance does not confer a utility allowance for a household with no elderly or disabled members, so there is no shelter deduction and the benefit is $407 a month."
+us,scenario_012,snap,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It counted the unborn child as a fourth member and applied a $973 four-person maximum. It also added a Mississippi utility allowance because of the energy subsidy. Its submitted $5,136 does not even match its own $7,968 result. The SNAP unit is 3 people, with no shelter deduction and a $785 maximum."
+us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It counted the pregnancy as a fourth member, using a four-person maximum of about $994, and added a utility allowance tied to the energy subsidy. The correct SNAP unit is 3 people with a $785 maximum and no shelter deduction."
+us,scenario_012,snap,claude-sonnet-5.5,llm_error,other,False,"It used the correct deductions and maximum but rounded the result to about $408 a month. SNAP rounds 30% of the $1,258 net income ($377.40) up to $378, giving $407 a month."
+us,scenario_012,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,It used a $211 standard deduction and a $786 maximum allotment instead of the FY2026 values of $209 and $785. That produced $409 a month instead of $407.
+us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It added a $500 utility allowance to rent, creating a $316.67 excess shelter deduction, and used a $200 standard deduction. The energy subsidy does not confer a utility allowance for this household, so net income stays at $1,258 and the benefit is $407 a month."
+us,scenario_012,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the FY2025 maximum allotment of $768 and standard deduction of $204 instead of the FY2026 values of $785 and $209. It also skipped rounding net income and the contribution, so it got $389.20 a month instead of $407."
+us,scenario_012,snap,deepseek-v4.1-flash,llm_error,household_unit_or_filing_status,False,"It counted the unborn child and used a 4-person SNAP unit and its larger maximum allotment. The SNAP unit is the 3 listed people, with a $785 maximum, giving $407 a month."
+us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It said the energy subsidy triggered a standard utility allowance and built an excess shelter deduction from it, giving about $482 a month. With no utility allowance, the $450 rent is below half of the $1,258 net income, so there is no shelter deduction and the benefit is $407 a month."
+us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave $446 a month with no derivation, $39 more than the correct $407. That overstatement fits a shelter deduction that does not apply, because the $450 rent is below half of the $1,258 net income."
+us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It applied an excess shelter deduction using a standard utility allowance. The energy subsidy does not confer one for this non-elderly, non-disabled household under 2026 rules, so net income is $1,258 and the benefit is $407 a month."
+us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied a shelter deduction that pushed the benefit to $457 a month. Rent alone ($450) is below half of the $1,258 net income, and no utility allowance applies, so the excess shelter deduction is zero and the benefit is $407 a month."
+us,scenario_012,snap,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation, and its $1,450 a year (about $121 a month) is far below the correct $785 − $378 = $407 a month. That fits leaving out the standard and earned income deductions, or using the wrong maximum allotment."
+us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It included an excess shelter deduction, giving $453 a month. Shelter costs are only the $450 rent, below half of the $1,258 net income, so there is no shelter deduction and the benefit is $407 a month."
+us,scenario_012,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It lowered net income with a shelter-cost deduction that does not exist here. Rent of $450 is below half of the $1,258 net income, and the energy subsidy gives no utility allowance, so the benefit is $407 a month."
+us,scenario_012,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the FY2025 maximum allotment ($768 a month, $9,216 a year) and a $200 standard deduction instead of the FY2026 values of $785 and $209. That gave $388 a month instead of $407."
+us,scenario_012,snap,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It gave a LIHEAP-triggered standard utility allowance and so an excess shelter deduction. Under 2026 rules, that pathway does not apply to a household with no elderly or disabled members, so there is no shelter deduction and the benefit is $407 a month."
+us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"It added a fourth member for the pregnancy, using a $975 maximum and a $238 standard deduction, and added a $360 utility allowance because of the energy subsidy. The SNAP unit is 3 people with a $785 maximum, a $209 standard deduction, and no shelter deduction."
+us,scenario_012,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"It treated the $411 energy subsidy as triggering a utility allowance and built a shelter deduction from it, which cut net income to about $1,044. Without a utility allowance, net income is $1,258 and the benefit is $407 a month."
+us,scenario_012,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It counted housing and utility costs, but there are no deductible utility costs and the $450 rent is below half of the $1,258 net income. With no shelter deduction, the benefit is $407 a month, not $497."
+us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It returned 0, claiming more eligibility inputs were needed. The facts fully support the calculation: a 3-person unit with $1,833 monthly earnings passes both income tests and receives $407 a month."
+us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It said the energy subsidy qualified the household for the heating/cooling utility allowance and so cut net income to about $1,000. Under 2026 rules, that does not apply to a household with no elderly or disabled members, so net income is $1,258 and the benefit is $407 a month."
+us,scenario_012,snap,gpt-5.6-luna,llm_error,other,False,"It correctly used no utility deduction but ended at $408 a month. SNAP rounds 30% of the $1,258 net income up to $378, so the benefit is $785 − $378 = $407 a month."
+us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It deducted a heating/cooling utility allowance along with rent and so created an excess shelter deduction. The energy subsidy does not confer that allowance for this household, leaving no shelter deduction and a $407 monthly benefit."
+us,scenario_012,snap,gpt-5.6-terra,llm_error,other,False,"It ended at $408 a month instead of $407 because it skipped the step that rounds the 30% contribution up: 30% of $1,258 is $377.40, which rounds up to $378."
+us,scenario_012,snap,gpt-6-astra,llm_error,other,False,"It applied the 2026 utility-allowance rule and every deduction correctly but worked on unrounded annual totals. SNAP rounds monthly net income to $1,258 and the 30% contribution up to $378, so the benefit is $407 × 12 = $4,884."
+us,scenario_012,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $406.20 a month implies 30% of $1,262.67, which means it used the FY2025 $204 standard deduction instead of the FY2026 $209. It also skipped the rounding steps."
+us,scenario_012,snap,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It included a shelter deduction, giving $504 a month. Rent of $450 is below half of the $1,258 net income, and no utility allowance applies, so there is no shelter deduction and the benefit is $407 a month."
+us,scenario_012,snap,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It added a $433 heating utility allowance because of the energy subsidy. Under 2026 rules, energy assistance does not confer a utility allowance on a household with no elderly or disabled members, so there is no shelter deduction and the benefit is $407 a month."
+us,scenario_012,snap,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It included an estimated utility allowance that created a shelter deduction. With rent alone below half of the $1,258 net income and no utility allowance, the benefit is $407 a month, not about $477."
+us,scenario_012,snap,grok-4.3,llm_error,other,False,"It gave about $300 a month with no derivation, well below the correct $407 (the $785 maximum minus the $378 contribution on $1,258 net income). It never carried out the deduction or allotment steps."
+us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"It used an outdated $198 standard deduction and $766 maximum allotment instead of the FY2026 values of $209 and $785, giving $385 a month instead of $407."
+us,scenario_012,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"It applied a LIHEAP-triggered standard utility allowance and so an excess shelter deduction. That pathway does not apply here under 2026 rules, so net income is $1,258 and the benefit is $407 a month, not $456."
+us,scenario_012,snap,grok-4.7,llm_error,other,False,"It calculated on annual figures and never applied SNAP's monthly rounding. Net income rounds to $1,258 and the 30% contribution rounds up to $378, giving $407 × 12 = $4,884, not $4,892.40."
+us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"It subtracted a $210 monthly standard deduction from annual income instead of the annual $2,508, which overstated annual net income at $17,390 instead of $15,096. It also counted the energy subsidy as a shelter cost and used a $9,422 annual maximum."
+us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no usable SNAP value or reasoning, so nothing was scored against the $4,884 reference."
+us,scenario_012,snap,kimi-k3,llm_error,other,False,"The deductions and maximum were correct, but it took 30% of the unrounded $1,257.67. SNAP rounds net income to $1,258 and the contribution up to $378, giving $407 a month, not $407.70."
+us,scenario_012,snap,minimax-m3,llm_error,categorical_eligibility,False,It returned 0 without doing any calculation. The household passes both the gross and net income tests and receives $785 − $378 = $407 a month.
+us,scenario_012,snap,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It gave a standard utility allowance because of the $411 energy subsidy, which lowered net income and pushed the benefit to $440 a month. With no utility allowance and no excess shelter cost, the benefit is $407 a month."
+us,scenario_012,snap,muse-wonder,llm_error,thresholds_rates,False,"It used the FY2025 three-person maximum allotment of $768 instead of the FY2026 $785, so it got about $391 a month instead of $407."
+us,scenario_012,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"It added a LIHEAP-funded utility allowance of about $470 to rent and produced a $290 excess shelter deduction. That allowance does not apply to this household under 2026 rules, so net income stays at $1,258 and the benefit is $407 a month."
+us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the $411 energy subsidy as unearned income and used outdated parameters: a $768 maximum and a $198 standard deduction. It then raised its own $4,525 result to $5,111 without justification. The correct figures are $1,258 net income and a $785 maximum, giving $407 a month."
+us,scenario_012,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It treated gross earnings of $22,000 as net income, leaving out the 20% earned income deduction and the $209 standard deduction, and so wrongly failed the household on the net income test. Correct net income is $1,258 a month, well under the poverty line, and the benefit is $407 a month."
us,scenario_012,spouse_chip_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated pregnancy and income below a pregnancy-coverage threshold as sufficient for CHIP. It omitted the prior Medicaid screen: the 18-year-old spouse qualifies for Medicaid under the OLDER_CHILD category, which makes her ineligible for CHIP."
us,scenario_012,spouse_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that pregnancy creates CHIP eligibility regardless of age once income requirements are met. It failed to apply the rule excluding anyone already Medicaid-eligible; this spouse is Medicaid-eligible as an OLDER_CHILD.
us,scenario_012,spouse_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model classified low-income pregnancy coverage as CHIP-funded coverage without first testing Medicaid eligibility. The spouse's Medicaid eligibility under the OLDER_CHILD category precludes CHIP, irrespective of household size adjustments for the unborn child or pregnancy income limits."
@@ -1062,57 +1074,58 @@ us,scenario_013,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,t
us,scenario_013,head_medicaid_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model asserted that income was ""near SSI/FPL-based aged-disabled standards"" but never compared it to a limit. The head's countable income is about $30,5xx: $23,736 of Social Security plus the $6,720 pension and $88 of dividends and capital gains, less the $20 general disregard. That is about 1.9x FPL, well above the 100% FPL aged/disabled income standard and the SSI benefit rate. SSI is 0, so no SSI-linked pathway applies, and at age 80 the head is outside the MAGI adult group."
us,scenario_013,head_medicaid_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model said Social Security is ""mostly excluded via SSI methodology"" and cut it to $10,196 countable. SSI methodology counts Social Security in full as unearned income, minus only the $20 general disregard, so countable income is about $30,5xx, or 1.9x FPL, above the aged/disabled standard. The model also applied the ALTCS 300%-of-SSI limit, which requires an institutional level-of-care need this household does not have. It then deducted health premiums and OTC costs as a spend-down, which does not create eligibility here, and it ignored its own finding that the $58,700 in assets exceeds the $2,000 limit."
us,scenario_013,head_medicaid_eligible,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model tested AGI of $6,808 against 138% FPL. MAGI adds back the $23,736 of nontaxable Social Security, which puts income at about $30,544, or 1.91x FPL. The 138% FPL adult expansion group also covers only ages 19-64, so it cannot apply to an 80-year-old. The aged/disabled pathway counts total income including Social Security, and that income is far above its limit."
-us,scenario_013,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test and a $4,500 elderly/disabled resource limit as if they were binding. It missed Arizona's 200% FPL BBCE from March 2026: gross income of $2,539 is under $2,608, which waives both tests, and the one-person household then gets the $24 minimum for 10 months ($240)."
-us,scenario_013,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Treated the 100% FPL net income test as the deciding test. It never applied Arizona's 200% FPL expanded categorical eligibility, which this household meets from March 2026 at $2,539 a month gross. That waives the net income test and gives the $24 one-person minimum allotment for March–December."
-us,scenario_013,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Applied a 130% FPL gross income test, which does not apply to elderly/disabled households, and a $2,500 asset limit. It missed that Arizona's 200% FPL BBCE (from March 2026) makes the household categorically eligible, so it gets the $24 minimum monthly benefit for 10 months."
-us,scenario_013,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Denied the household on the regular aged/disabled income and asset tests. It missed that from March 2026 Arizona's BBCE at 200% FPL treats those tests as met for gross income of $2,539 a month, which leaves the $24 one-person minimum allotment for 10 months."
-us,scenario_013,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"Ended at the 100% FPL net income test. It skipped Arizona's 200% FPL expanded categorical eligibility, which this household meets from March 2026 and which waives the net income test. The negative formula benefit is then replaced by the $24 minimum for a one-person household, for $240 a year."
-us,scenario_013,snap,claude-opus-4.8,llm_error,asset_resource,False,"Disqualified the household on the $4,500 elderly/disabled resource limit. Arizona's BBCE at 200% FPL (from March 2026) removes the asset test for households with gross income under $2,608 a month, so the $58,700 in the bank does not matter and the household gets the $24 minimum for 10 months."
-us,scenario_013,snap,claude-opus-5,llm_error,asset_resource,False,"Correctly skipped the gross income test, then denied the household on the regular SNAP asset limit. Under Arizona's 200% FPL BBCE from March 2026, the asset test is treated as met, so the household qualifies for the $24 one-person minimum allotment for March–December."
-us,scenario_013,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income limit as binding. It missed that Arizona's expanded categorical eligibility (200% FPL from March 2026, met at $2,539 a month gross) waives that test, and that categorically eligible one-person households receive the $24 minimum benefit."
-us,scenario_013,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Denied the household on the net income and $4,500 resource tests, and also misread the annual $298 premium as a monthly cost. It missed that Arizona's 200% FPL BBCE from March 2026 waives both tests, which yields the $24 minimum allotment for 10 months."
-us,scenario_013,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Found the 200% FPL BBCE pathway but used a stale limit of about $2,510 a month instead of $2,608.33 (2025 guideline). It also put $72 of capital gains into gross income, which is not countable SNAP income, giving $2,545 instead of $2,539. With the right figures the household is under the limit from March 2026 and gets the $24 minimum for 10 months."
-us,scenario_013,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Stopped at the $298 maximum less 30% of net income, which is negative, and returned zero. A one-person household made categorically eligible by Arizona's 200% FPL BBCE from March 2026 receives the $24 minimum allotment instead, for $240 a year."
-us,scenario_013,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"Applied the regular elderly/disabled asset limit to the $58,700 in the bank. Arizona's expanded categorical eligibility at 200% FPL (from March 2026) treats the asset test as met at $2,539 a month gross, so the household gets the $24 minimum for 10 months."
-us,scenario_013,snap,deepseek-v4-pro,llm_error,asset_resource,False,"Applied the $4,250 elderly/disabled resource limit as disqualifying. It missed Arizona's 200% FPL BBCE from March 2026, which waives the resource test for this household and gives the $24 one-person minimum benefit for March–December."
-us,scenario_013,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Denied the household on both the net income limit and the elderly/disabled resource limit. Arizona's BBCE at 200% FPL from March 2026 treats both tests as met for gross income of $2,539 a month, which leaves the $24 minimum allotment for 10 months."
-us,scenario_013,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Applied the regular net income and asset tests. It never checked Arizona's 200% FPL expanded categorical eligibility, which this household meets from March 2026; that waives those tests and gives the $24 one-person minimum benefit, $240 a year."
-us,scenario_013,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Used a 185% FPL gross limit of about $2,321 and counted $72 of capital gains. From March 2026 Arizona's BBCE limit is 200% FPL ($2,608.33), and countable gross income is $2,539. The household therefore qualifies categorically and gets the $24 minimum, not zero."
-us,scenario_013,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Denied the household on the regular resource limit and net income test. It missed that Arizona's 200% FPL BBCE from March 2026 treats both tests as met, so the household receives the $24 one-person minimum allotment for 10 months."
-us,scenario_013,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Applied the elderly net income limit as decisive. Arizona's expanded categorical eligibility (gross under 200% FPL, $2,608.33, from March 2026) waives that limit, and the categorically eligible one-person household gets the $24 minimum benefit."
-us,scenario_013,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"Used 185% FPL as Arizona's BBCE limit and counted capital gains, giving $30,544 of income. From March 2026 the limit is 200% FPL ($2,608.33 a month) and countable gross income is $2,539. The asset test is therefore waived and the household gets the $24 minimum for 10 months."
-us,scenario_013,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Applied the regular income and asset thresholds. It missed that Arizona's 200% FPL BBCE from March 2026 treats them as met for gross income of $2,539 a month, which yields the $24 one-person minimum benefit for 10 months."
-us,scenario_013,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"Claimed gross income exceeds Arizona's eligibility threshold. From March 2026 Arizona's BBCE gross limit is 200% FPL ($2,608.33 a month), which is above the $2,539 countable gross income, so the household is categorically eligible and gets the $24 minimum allotment."
-us,scenario_013,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Applied the net income limit and took the negative formula benefit as zero. It missed that Arizona's 200% FPL BBCE from March 2026 confers eligibility, and that eligible one-person households receive the $24 minimum allotment."
-us,scenario_013,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used 185% FPL as Arizona's BBCE limit and counted capital gains in gross income. From March 2026 the limit is 200% FPL ($2,608.33) against $2,539 of countable income, so the asset test is waived and the household gets $24 a month for 10 months."
-us,scenario_013,snap,gemini-4-argon,llm_error,categorical_eligibility,False,"Denied the household on the elderly/disabled asset limit and the net income limit. It missed that Arizona's 200% FPL BBCE from March 2026 treats both tests as met, which gives the $24 one-person minimum benefit, $240 a year."
-us,scenario_013,snap,glm-5.2,llm_error,categorical_eligibility,False,Applied the 100% FPL net income limit as decisive. It missed that Arizona's expanded categorical eligibility at 200% FPL (met from March 2026) waives that test and gives the $24 minimum allotment for a one-person household.
-us,scenario_013,snap,glm-5.3,llm_error,categorical_eligibility,False,"Tested net income against the regular gross and net limits; the gross test does not apply to an elderly/disabled household. It missed Arizona's 200% FPL BBCE from March 2026, which makes the household categorically eligible for the $24 one-person minimum for 10 months."
-us,scenario_013,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Treated Social Security plus pension as too much income for any benefit. Gross income of $2,539 a month is under Arizona's 200% FPL BBCE limit from March 2026, so the household is categorically eligible and receives the $24 minimum allotment for 10 months."
-us,scenario_013,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Gave no substantive eligibility analysis and defaulted to zero. The correct derivation puts gross income of $2,539 a month under Arizona's 200% FPL BBCE limit from March 2026, which confers eligibility and the $24 one-person minimum for March–December, $240 in total."
-us,scenario_013,snap,gpt-5.5,llm_error,asset_resource,False,"Applied the elderly/disabled resource limit to the $58,700 in the bank. Arizona's 200% FPL expanded categorical eligibility from March 2026 waives the resource test for this household, and it gets the $24 minimum for 10 months."
-us,scenario_013,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Computed a zero allotment from the benefit formula. It missed that a one-person household categorically eligible through Arizona's 200% FPL BBCE (from March 2026) receives the $24 minimum benefit when the formula gives less, so the annual total is $240."
-us,scenario_013,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"Concluded the formula gives no positive allotment and stopped there. It never applied Arizona's 200% FPL BBCE from March 2026 or the $24 minimum allotment that eligible one-person households receive, which together give $240 a year."
-us,scenario_013,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Applied the net income limit as decisive. Arizona's expanded categorical eligibility (gross under 200% FPL from March 2026) treats that test as met, and the one-person household gets the $24 minimum for 10 months."
-us,scenario_013,snap,gpt-6-astra,llm_error,thresholds_rates,False,"Said income exceeds Arizona's categorical-eligibility limit. From March 2026 that limit is 200% FPL ($2,608.33 a month, $2,660 from October), above the $2,539 countable gross income, so the asset test is waived and the household gets the $24 minimum for 10 months."
-us,scenario_013,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"Took the negative formula result as a zero allotment. It missed that Arizona's 200% FPL BBCE from March 2026 makes the household eligible, and that eligible one-person households receive the $24 minimum benefit."
-us,scenario_013,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"Concluded the benefit formula produces no allotment. It never applied the $24 minimum for eligible one- and two-person households, which this household gets through Arizona's 200% FPL BBCE from March 2026, for $240 a year."
-us,scenario_013,snap,gpt-6.1-sol,llm_error,thresholds_rates,False,"Said income exceeds Arizona's expanded categorical eligibility limit. From March 2026 that limit is 200% FPL ($2,608.33 a month), above the $2,539 countable gross income, so the elderly/disabled asset limit is waived and the household gets the $24 minimum for 10 months."
-us,scenario_013,snap,gpt-6.1-terra,llm_error,categorical_eligibility,False,"Applied the net income limit as disqualifying. Arizona's 200% FPL BBCE from March 2026 treats the net income test as met, and the household receives the $24 one-person minimum allotment."
-us,scenario_013,snap,grok-4.3,llm_error,categorical_eligibility,False,"Denied the household on the regular income and asset limits. It missed that Arizona's 200% FPL expanded categorical eligibility from March 2026 treats both as met, which gives the $24 minimum for 10 months, $240."
-us,scenario_013,snap,grok-4.5,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income limit after deductions. It missed that Arizona's BBCE at 200% FPL from March 2026 waives that limit for gross income of $2,539, and that the eligible one-person household gets the $24 minimum benefit."
-us,scenario_013,snap,grok-4.6,llm_error,categorical_eligibility,False,"Denied the household on the net income limit and the elderly/disabled resource limit. Arizona's 200% FPL BBCE from March 2026 treats both tests as met, which gives the $24 one-person minimum allotment for 10 months."
-us,scenario_013,snap,grok-4.7,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the $298 maximum, then returned zero. It missed that a one-person household made categorically eligible by Arizona's 200% FPL BBCE from March 2026 receives the $24 minimum benefit instead."
-us,scenario_013,snap,grok-build-0.1,llm_error,asset_resource,False,"Applied the $4,250 elderly/disabled resource limit as disqualifying. Arizona's expanded categorical eligibility at 200% FPL (from March 2026) removes the asset test for this household, which gets the $24 minimum for March–December."
-us,scenario_013,snap,inkling,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income limit as decisive. It never applied Arizona's 200% FPL BBCE from March 2026, which waives that test, or the $24 minimum allotment for eligible one-person households."
-us,scenario_013,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"Denied the household on the federal elderly/disabled resource limit and the 100% FPL net income test. Arizona's 200% FPL BBCE from March 2026 treats both as met, giving the $24 one-person minimum for 10 months."
-us,scenario_013,snap,kimi-k3,llm_error,categorical_eligibility,False,"Applied the one-person net income limit as disqualifying. Arizona's expanded categorical eligibility (gross $2,539 under the $2,608.33 200% FPL limit from March 2026) waives that limit, and the household gets the $24 minimum benefit."
-us,scenario_013,snap,minimax-m3,llm_error,categorical_eligibility,False,"Said income and assets exceed SNAP thresholds. It missed that Arizona's 200% FPL BBCE from March 2026 treats both tests as met for this household, which gives the $24 one-person minimum allotment for 10 months."
-us,scenario_013,snap,muse-spark-1.3,llm_error,asset_resource,False,"Applied the roughly $4,250 elderly/disabled asset limit. Arizona's 200% FPL BBCE from March 2026 waives the asset test at $2,539 of monthly gross income, so the household gets the $24 minimum for 10 months."
-us,scenario_013,snap,muse-wonder,llm_error,categorical_eligibility,False,"Denied the household on the net income limit and the $58,700 in bank assets. It missed that Arizona's 200% FPL expanded categorical eligibility from March 2026 treats both tests as met, giving the $24 one-person minimum benefit, $240 a year."
-us,scenario_013,snap,ox-alpha,llm_error,categorical_eligibility,False,"Correctly exempted the household from the gross income test but treated the 100% FPL net income test as binding. Arizona's 200% FPL BBCE from March 2026 waives the net income test, so the household receives the $24 minimum for 10 months."
-us,scenario_013,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross income test, which does not apply to an elderly/disabled household. It also missed Arizona's 200% FPL BBCE from March 2026, which makes the household categorically eligible for the $24 one-person minimum for 10 months."
-us,scenario_013,snap,qwen3.8-max,llm_error,age_disability,False,"Made up a rule that members aged 60 or older are categorically ineligible for SNAP. In fact elderly households get favorable treatment, and this one qualifies through Arizona's 200% FPL BBCE from March 2026 for the $24 minimum allotment, $240 a year."
+us,scenario_013,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It applied only the ordinary elderly/disabled tests (100% FPL net income and the ~$4,500 resource limit) and never checked Arizona's BBCE. From March 2026, BBCE covers gross income up to 200% FPL ($2,608.33/mo), so this ~$2,539 household passes, the asset and net income tests count as met, and the one-person household gets the $24 minimum for 10 months ($240)."
+us,scenario_013,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It ruled the household out on the 100% FPL net income test without checking Arizona's BBCE. From March 2026, BBCE confers eligibility at gross income up to 200% FPL ($2,608.33/mo), which removes the net income test. The categorically eligible one-person household then gets the $24 minimum for 10 months instead of $0."
+us,scenario_013,snap,claude-fable-5.2,llm_error,state_local_rule,False,"It used a 185% FPL limit for Arizona's BBCE, but from March 2026 the limit is 200% FPL ($2,608.33/mo). The ~$2,539 gross income qualifies, so the asset and net income tests count as met. The one-person household gets the $24 minimum for March through December ($240)."
+us,scenario_013,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross income test (elderly/disabled households are exempt from it) and a $2,500 asset limit. It missed Arizona's BBCE at 200% FPL from March 2026, which waives the asset test for this ~$2,539/mo household and gives the $24 one-person minimum for 10 months."
+us,scenario_013,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It denied benefits on the aged/disabled income and asset tests without applying Arizona's BBCE. From March 2026, gross income up to 200% FPL ($2,608.33/mo) confers categorical eligibility, so both tests count as met, and the one-person household gets the $24 minimum for 10 months ($240)."
+us,scenario_013,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"It treated the 100% FPL net income test as decisive and never checked Arizona's BBCE. From March 2026, BBCE confers eligibility because gross income (~$2,539/mo) is under 200% FPL ($2,608.33). As a categorically eligible one-person household, it gets the $24 minimum allotment for 10 months."
+us,scenario_013,snap,claude-opus-4.8,llm_error,asset_resource,False,"It disqualified the household for having $58,700 in bank assets, which is over the ~$4,500 elderly/disabled limit. It missed that Arizona's BBCE, which from March 2026 covers gross income up to 200% FPL ($2,608.33/mo), eliminates the asset test. The household therefore gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,claude-opus-5,llm_error,asset_resource,False,"It applied the regular SNAP asset limit to the $58,700 balance. This ~$2,539/mo household falls under Arizona's 200% FPL BBCE threshold from March 2026, which treats the asset test as met. The one-person household gets the $24 minimum for March through December ($240)."
+us,scenario_013,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"It ruled the household out on the 100% FPL net income limit and missed Arizona's BBCE at 200% FPL from March 2026, which removes the net income test. The categorically eligible one-person household gets the $24 minimum allotment for 10 months even though the benefit formula is negative."
+us,scenario_013,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It applied the 130% gross income test, the net income test and the $4,500 asset test. It also treated the annual $298 premium as a monthly expense. It never applied Arizona's BBCE, which from March 2026 confers eligibility at gross income up to 200% FPL ($2,608.33/mo) and gives this one-person household the $24 minimum for 10 months."
+us,scenario_013,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It cited a 200% FPL limit but understated it as ~$2,510/mo. The actual limit is $2,608.33/mo (2 × $15,650 / 12), which the household's ~$2,539 SNAP income clears. It also kept applying the $4,250 asset limit, which BBCE eliminates. The household gets the $24 one-person minimum from March through December."
+us,scenario_013,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It took the benefit as $298 minus 30% of net income with a floor of zero, missing that categorically eligible one- and two-person households receive the minimum allotment. Under Arizona's 200% FPL BBCE from March 2026, this household is categorically eligible and gets $24 a month for 10 months."
+us,scenario_013,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It denied SNAP on the elderly/disabled asset limit and ignored Arizona's BBCE. From March 2026, BBCE confers eligibility at gross income up to 200% FPL ($2,608.33/mo) and treats the asset test as met, so the household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,deepseek-v4-pro,llm_error,asset_resource,False,"It applied the $4,250 elderly/disabled resource limit to the $58,700 balance. Arizona's BBCE, which from March 2026 covers gross income up to 200% FPL, makes this household categorically eligible with no asset test. It gets the $24 one-person minimum for 10 months ($240)."
+us,scenario_013,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It applied the ordinary net income test and the elderly/disabled resource limit and missed Arizona's BBCE. From March 2026, BBCE covers gross income (~$2,539/mo) below 200% FPL ($2,608.33), so both tests count as met, and the one-person household gets the $24 minimum for 10 months."
+us,scenario_013,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It ruled the household out on the net income and asset limits without checking Arizona's BBCE. From March 2026, BBCE confers eligibility at gross income up to 200% FPL and treats those tests as met, so the one-person household gets the $24 minimum for 10 months."
+us,scenario_013,snap,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used a 185% FPL (~$2,321/mo) gross income limit for Arizona, but from March 2026 Arizona's BBCE limit is 200% FPL ($2,608.33/mo). The household's SNAP gross income ($2,539.33) qualifies. It also assumed a zero formula result means no benefit, missing the $24 minimum for categorically eligible one-person households."
+us,scenario_013,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It applied the regular resource limit and net income test and missed Arizona's BBCE at 200% FPL from March 2026. BBCE treats both tests as met for this ~$2,539/mo household and gives the $24 one-person minimum for 10 months."
+us,scenario_013,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It treated the elderly net income limit as disqualifying and never applied Arizona's BBCE. From March 2026, BBCE confers eligibility at gross income up to 200% FPL ($2,608.33/mo), removing the net income test, so the one-person household gets the $24 minimum for 10 months."
+us,scenario_013,snap,gemini-3.5-flash,llm_error,state_local_rule,False,"It took Arizona's BBCE limit as 185% FPL and then applied the $4,250 asset limit. From March 2026, Arizona's BBCE limit is 200% FPL ($2,608.33/mo), which the household's $2,539.33 SNAP income clears, so the asset test is waived. The household gets $24 a month for 10 months."
+us,scenario_013,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It said income and assets exceed SNAP limits without applying Arizona's BBCE. From March 2026, BBCE confers eligibility at gross income up to 200% FPL and treats the income and asset tests as met, so the one-person household gets the $24 minimum for 10 months."
+us,scenario_013,snap,gemini-3.6-flash,llm_error,state_local_rule,False,"It claimed gross income exceeds Arizona's threshold. From March 2026, Arizona's BBCE gross limit is 200% FPL ($2,608.33/mo), above the household's $2,539.33. The household is categorically eligible and gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"It treated the net income limit and the zero formula result as final, missing Arizona's 200% FPL BBCE from March 2026. BBCE makes the household categorically eligible, so it gets the $24 minimum allotment for one-person households in each of those 10 months."
+us,scenario_013,snap,gemini-3.8-flash,llm_error,state_local_rule,False,"It used 185% FPL as Arizona's BBCE limit and then applied the elderly asset limit. From March 2026, the limit is 200% FPL ($2,608.33/mo), so the $2,539.33 SNAP gross income confers categorical eligibility with no asset test. The household gets the $24 minimum for 10 months."
+us,scenario_013,snap,gemini-4-argon,llm_error,categorical_eligibility,False,"It denied SNAP on the elderly/disabled asset limit and net income limit and missed Arizona's BBCE at 200% FPL from March 2026, which treats both tests as met. The one-person household then gets the $24 minimum allotment for 10 months."
+us,scenario_013,snap,glm-5.2,llm_error,categorical_eligibility,False,"It compared annual income with the ~100% FPL net income limit and stopped there. Arizona's BBCE, from March 2026, confers eligibility at gross income up to 200% FPL ($31,300/yr), which this household meets. As a one-person household, it gets the $24 minimum for 10 months."
+us,scenario_013,snap,glm-5.3,llm_error,categorical_eligibility,False,"It computed net income against the senior gross and net income limits, wrongly counted capital gains as income, and never applied Arizona's BBCE. From March 2026, BBCE covers gross income up to 200% FPL and gives this categorically eligible one-person household the $24 minimum for 10 months."
+us,scenario_013,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It judged Social Security plus pension income too high without checking Arizona's BBCE. From March 2026, BBCE covers gross income up to 200% FPL ($2,608.33/mo), and a categorically eligible one-person household receives the $24 minimum, which gives $240 for 10 months."
+us,scenario_013,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It wrongly asserted that no inputs supported a benefit and never ran any eligibility test. The household's $2,539.33 monthly gross income is under Arizona's 200% FPL BBCE limit from March 2026, so it is categorically eligible and gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,gpt-5.5,llm_error,asset_resource,False,"It applied the elderly/disabled resource limit to the $58,700 balance. Arizona's BBCE, from March 2026 at 200% FPL gross income, treats the asset test as met for this household, so it receives the $24 one-person minimum for 10 months."
+us,scenario_013,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It took a zero allotment from the benefit formula as final. Under Arizona's 200% FPL BBCE from March 2026 the household is categorically eligible, and categorically eligible one-person households get the $24 minimum allotment even when the formula is below zero."
+us,scenario_013,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It concluded that income is too high for a positive allotment and missed the one-person minimum benefit under BBCE. Arizona's 200% FPL BBCE from March 2026 makes this household categorically eligible, so it gets $24 a month for 10 months."
+us,scenario_013,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It applied the net income limit and ignored Arizona's BBCE. From March 2026, BBCE confers eligibility at gross income up to 200% FPL ($2,608.33/mo) and treats the net income test as met, so the one-person household gets the $24 minimum for 10 months."
+us,scenario_013,snap,gpt-6-astra,llm_error,state_local_rule,False,"It said income exceeds Arizona's categorical eligibility limit, which reflects the old 185% FPL threshold. From March 2026, Arizona's BBCE limit is 200% FPL ($2,608.33/mo), above the household's $2,539.33. The asset test is then waived, and the household gets the $24 minimum for 10 months."
+us,scenario_013,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"It treated the zero formula allotment as the answer. It missed that Arizona's 200% FPL BBCE from March 2026 makes the household categorically eligible, which entitles this one-person household to the $24 minimum each month from March through December."
+us,scenario_013,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It concluded that income produces no allotment and missed the one-person minimum benefit. Under Arizona's BBCE, which from March 2026 confers eligibility at gross income up to 200% FPL, this household is categorically eligible and gets $24 a month for 10 months."
+us,scenario_013,snap,gpt-6.1-sol,llm_error,state_local_rule,False,"It said income exceeds Arizona's expanded categorical eligibility limit, which reflects the old 185% FPL threshold. From March 2026, the limit is 200% FPL ($2,608.33/mo), which the $2,539.33 SNAP gross income clears, so the asset test is waived. The household gets $24 a month for 10 months."
+us,scenario_013,snap,gpt-6.1-terra,llm_error,categorical_eligibility,False,"It applied the net income limit and missed Arizona's BBCE at 200% FPL from March 2026, which treats that test as met. The categorically eligible one-person household gets the $24 minimum allotment for 10 months."
+us,scenario_013,snap,grok-4.3,llm_error,categorical_eligibility,False,"It said income and assets exceed SNAP limits without applying Arizona's BBCE. From March 2026, BBCE confers eligibility at gross income up to 200% FPL and treats the income and asset tests as met, so the household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,grok-4.5,llm_error,categorical_eligibility,False,"It ruled the household out on the 100% FPL net income limit and missed Arizona's BBCE at 200% FPL from March 2026, which removes that test. The categorically eligible one-person household gets the $24 minimum for 10 months."
+us,scenario_013,snap,grok-4.6,llm_error,categorical_eligibility,False,"It applied the net income limit and the elderly/disabled resource limit and never checked Arizona's BBCE. From March 2026, BBCE confers eligibility at gross income up to 200% FPL ($2,608.33/mo) and waives both tests, so the household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,grok-4.7,llm_error,categorical_eligibility,False,"It stopped once 30% of net income exceeded the $298 maximum allotment. It missed that the household is categorically eligible under Arizona's 200% FPL BBCE from March 2026, so as a one-person household it receives the $24 minimum allotment for 10 months."
+us,scenario_013,snap,grok-build-0.1,llm_error,asset_resource,False,"It applied the $4,250 elderly/disabled resource limit to the $58,700 balance. Arizona's BBCE, from March 2026 at 200% FPL gross income, treats the asset test as met for this household, which gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,inkling,llm_error,categorical_eligibility,False,"It ruled the household out on the 100% FPL net income limit and never applied Arizona's BBCE. From March 2026, BBCE confers eligibility at gross income up to 200% FPL ($2,608.33/mo), so the one-person household gets the $24 minimum for 10 months."
+us,scenario_013,snap,kimi-k2.6,llm_error,categorical_eligibility,False,It applied the elderly/disabled resource limit and the 100% FPL net income test and missed Arizona's BBCE at 200% FPL from March 2026. BBCE treats both tests as met and gives the one-person household the $24 minimum for 10 months.
+us,scenario_013,snap,kimi-k3,llm_error,categorical_eligibility,False,"It denied SNAP on the net income limit without checking Arizona's BBCE. From March 2026, BBCE confers eligibility at gross income up to 200% FPL ($2,608.33/mo) and removes the net income test, so the one-person household gets the $24 minimum for 10 months."
+us,scenario_013,snap,minimax-m3,llm_error,categorical_eligibility,False,"It said income and assets exceed the elderly SNAP thresholds and ignored Arizona's BBCE. From March 2026, BBCE covers gross income up to 200% FPL and treats the income and asset tests as met, so the household gets $24 a month for 10 months."
+us,scenario_013,snap,muse-spark-1.3,llm_error,asset_resource,False,"It applied the ~$4,250 elderly/disabled asset limit and missed that Arizona's BBCE, from March 2026 at 200% FPL gross income, eliminates the asset test. The household then gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,muse-wonder,llm_error,categorical_eligibility,False,"It applied the net income limit and the asset test without checking Arizona's BBCE. From March 2026, BBCE confers eligibility at gross income up to 200% FPL ($2,608.33/mo) and waives both tests, so the household gets the $24 one-person minimum for 10 months."
+us,scenario_013,snap,ox-alpha,llm_error,categorical_eligibility,False,"It used the elderly/disabled exemption from the gross income test, then failed the household on the 100% FPL net income test, missing Arizona's BBCE at 200% FPL from March 2026. BBCE removes the net income test, and the one-person household gets the $24 minimum for 10 months."
+us,scenario_013,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"It applied the 130% FPL gross income test, which does not apply to elderly/disabled households, and missed Arizona's BBCE at 200% FPL from March 2026. This household's gross income is under that limit, so it is categorically eligible for the $24 one-person minimum for 10 months."
+us,scenario_013,snap,qwen3.8-max,llm_error,age_disability,False,"It invented a rule that people aged 60 or older are categorically ineligible for SNAP. Elderly households qualify under special rules, and under Arizona's 200% FPL BBCE from March 2026 this household is categorically eligible for the $24 one-person minimum for 10 months."
us,scenario_013,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model applied Arizona’s 2.5% rate to $88 of dividends and capital gains without subtracting the Arizona standard deduction, which reduces taxable income to zero. It also violated its own computation by submitting $744.10 after its explanation calculated $2.20."
us,scenario_013,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model claimed Arizona offers no refundable individual credits. It missed the refundable Increased Excise Tax Credit, which pays $25 to a single filer with federal AGI at or below $12,500. This filer's AGI is $6,808, so the credit is $25."
us,scenario_013,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model said no Arizona refundable credit applies and never considered the Increased Excise Tax Credit. That credit pays $25 per taxpayer when a single filer's AGI is at or below $12,500, and this filer's AGI is $6,808."
@@ -1211,57 +1224,58 @@ us,scenario_014,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The
us,scenario_014,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there is no answer to grade. The correct derivation is 7.65% of the spouse's $88,928 wages, about $6,803."
us,scenario_014,payroll_tax,minimax-m3,llm_error,other,False,"The model's explanation computes $5,513.54 Social Security plus $1,289.46 Medicare, about $6,803, on the spouse's $88,928 wages. It then submitted 'value = 6581', a number that contradicts its own sum."
us,scenario_014,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated the correct components, $5,513.54 Social Security and $1,289.46 Medicare on $88,928 with no Additional Medicare Tax. It then submitted $8,075.02 instead of their $6,802.99 sum, adding about $1,272 that its own explanation never supports."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Correctly reached $84,928 of taxable income but used the 2025 rates (2.22%–4.82%) instead of the 2026 schedule (2.11%/2.81%/3.16%/4.22%/4.58%). It then raised its own $3,256 to $3,374 with an unexplained 'rounding' adjustment. The 2026 rates give $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Had the right $84,928 base and bracket widths but used rates of 2.13%/2.84%/3.20%/4.26%/4.63%. Those rates are too high; the 2026 schedule is 2.11%/2.81%/3.16%/4.22%/4.58%. The higher rates overstate tax by about $33 (correct figure: $3,092.19)."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Subtracted the $14,717 ESI premium from wages even though PolicyEngine keeps WV AGI at $88,927.65. It also never applied the $4,000 personal exemption and used a flat ~4.5% effective rate instead of the graduated 2026 schedule (2.11%–4.58%), which yields $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"Gave a rough $2,700 with no computation. That is about a 3.2% effective rate on $84,928, below the 3.64% that the 2026 graduated schedule (2.11%/2.81%/3.16%/4.22%/4.58%) produces, so the answer understates the $3,092.19 liability."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Computed $3,255 using the 2025 rates (2.22%–4.82%) instead of the 2026 rates (2.11%–4.58%). It then subtracted about $378 of undefined 'nonrefundable credits' that do not exist for this childless household at $89k AGI."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Used a mismatched set of rates (2.2%/2.96%/3.33%/4.07%/4.625%) to reach $3,130. It then applied an arbitrary further 'rate cut adjustment' to get $2,849 instead of applying the actual 2026 schedule (2.11%/2.81%/3.16%/4.22%/4.58%), which gives $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Had the right $84,928 taxable income but reported $1,655, roughly a 1.95% effective rate. That is far below the 2026 graduated schedule, which reaches 4.58% above $60k and totals $3,092.19, and no nonrefundable credits apply."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Correctly computed $84,928 of taxable income but applied the 2025 rates (2.22%/2.96%/3.33%/4.44%/4.82%) instead of the lower 2026 rates (2.11%/2.81%/3.16%/4.22%/4.58%). That gives $3,255.03 instead of $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Used outdated rates (2.36%/3.15%/3.54%/4.72%) to reach $3,360. It then arbitrarily cut that to $1,919 by citing credits it had already said do not apply. The 2026 schedule (2.11%–4.58%) on $84,928 gives $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Had the right $84,928 base but rounded to $3,200 based on a vague 2.36%–5.12% / 4.82% top-rate range. It never applied the 2026 schedule (2.11%/2.81%/3.16%/4.22%/4.58%), which yields $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Assumed the 2024 rates (2.36%/3.15%/3.54%/4.72%/5.12%) still apply in 2026, which gives $3,459.80. It missed the later rate cuts that bring the 2026 schedule down to 2.11%/2.81%/3.16%/4.22%/4.58% ($3,092.19)."
-us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Invented a $4,000 WV standard deduction and $6,000 per-person exemptions, which reduced taxable income to $72,928. WV has no standard deduction and allows only $2,000 per exemption, so taxable income is $84,927.65. It also used nonexistent 4.50%/5.16% upper rates."
-us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Subtracted the $14,717 ESI premium to get $74,211 of AGI (PolicyEngine's AGI is $88,927.65). It also applied a nonexistent $15,400 WV standard deduction instead of the $4,000 exemptions and used made-up rates up to 5.12%."
-us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Reduced AGI to $74,211 by subtracting the ESI premium and added a nonexistent $4,000 MFJ standard deduction, which shrank taxable income to $66,211 instead of $84,927.65. Its rates (2.18%–4.72%) were also above the 2026 schedule of 2.11%–4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Subtracted $8,000 of deductions and exemptions instead of the $4,000 of exemptions WV allows, and used invented rates (1.69%–4.29%) that sit below the 2026 schedule (2.11%–4.58%). The correct calculation gives $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the spouse's $14,717 ESI premium from wages, which produced $70,211 of taxable income instead of $84,927.65. It then applied the outdated 2024 rates (2.36%–5.12%) rather than the 2026 schedule of 2.11%–4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Gave $4,125 with no computation, which matches a flat ~4.6–4.9% top rate applied to all wages. The graduated 2026 schedule (2.11%/2.81%/3.16%/4.22%/4.58%) on $84,927.65 after $4,000 of exemptions yields $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Subtracted the $14,717 ESI premium to get $74,211 of AGI, so its taxable income was $70,211 instead of $84,927.65. It also applied 2024-level marginal rates instead of the 2026 schedule (2.11%–4.58%)."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Started from $74,211 of AGI by deducting the ESI premium, which understated taxable income at $70,211 versus $84,927.65. It also used pre-2026 rates rather than the 2026 schedule of 2.11%/2.81%/3.16%/4.22%/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Claimed deductions and exemptions eliminate all tax. WV allows only $4,000 of personal exemptions and no standard deduction, which leaves $84,927.65 taxable and $3,092.19 of tax."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Subtracted the ESI premium to get $74,211 of AGI and $70,211 of taxable income, instead of $84,927.65. It then applied the 2024 rates (2.36%–5.12%), which gives $2,706.30, not the 2026 schedule (2.11%–4.58%)."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Understated taxable income at $70,211 by deducting the $14,717 ESI premium; the correct base is $84,927.65. It then applied a rate schedule well above the 2026 rates (2.11%–4.58%)."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Had the right $84,928 base but applied the 2024 rates (2.36%/3.15%/3.54%/4.72%/5.12%). The 2026 rates are 2.11%/2.81%/3.16%/4.22%/4.58%, so its $3,459.81 should be $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"Applied the outdated 2024 rates (2.36%–5.12%) to the correct $84,928 base, which gives $3,459.81. It missed the reduced 2026 rates (2.11%–4.58%), which produce $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Invented an $8,000 WV standard deduction (WV has none) and applied the flat 4.82% top rate to all taxable income. It never used the graduated 2026 brackets (2.11% up to $10k through 4.58% above $60k)."
-us,scenario_014,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Calculated Kentucky tax on a fabricated $25,050 base with a Kentucky family size credit, although the household lives in West Virginia. WV tax on $84,927.65 at the 2026 rates is $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Gave $2,331 without a computation, about a 2.7% effective rate on $84,928. The 2026 WV schedule (2.11%/2.81%/3.16%/4.22%/4.58%) after $4,000 of exemptions yields $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Assumed deductions and credits wipe out WV tax. WV allows only $4,000 of personal exemptions, no standard deduction, and no nonrefundable credits for this household, so $84,927.65 is taxable and the tax is $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Applied the 2025 rates (2.22%/2.96%/3.33%/4.44%/4.82%) to the correct $84,928 base, giving $3,255.03. It missed the further reduction to 2026 rates of 2.11%/2.81%/3.16%/4.22%/4.58%, which gives $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Had the right $84,928 taxable income but used a rate schedule above the 2026 rates (2.11%/2.81%/3.16%/4.22%/4.58%). That overstates tax at $3,217.53 versus $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Used the 2025 schedule (2.22%–4.82%) as its 'projected' 2026 rates, giving $3,255.03. It missed the actual 2026 rates of 2.11%/2.81%/3.16%/4.22%/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Got the base right but its $3,255 matches the 2025 rates (2.22%–4.82%) instead of the lower 2026 schedule (2.11%–4.58%), which yields $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"Applied the 2025 rates (2.22%/2.96%/3.33%/4.44%/4.82%) to $84,928 instead of the 2026 rates (2.11%/2.81%/3.16%/4.22%/4.58%). That overstates the tax by about $163."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Had the correct $84,928 taxable income but used a schedule somewhat above the 2026 rates (2.11%–4.58%). The result is $3,192.03 instead of $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Used the 2025 joint rates (2.22%–4.82%) on $84,928 instead of the reduced 2026 rates (2.11%/2.81%/3.16%/4.22%/4.58%), producing $3,255.03 rather than $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"Had the right $84,928 base but applied rates below the 2026 schedule, giving a 3.48% effective rate. The 2026 rates (2.11%/2.81%/3.16%/4.22%/4.58%) produce $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"Its $3,255.03 matches the 2025 rates (2.22%–4.82%) applied to $84,928. It missed the 2026 rates of 2.11%/2.81%/3.16%/4.22%/4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Claimed a standard deduction eliminates WV tax. WV has no standard deduction, only $4,000 of personal exemptions, which leaves $84,927.65 taxable and $3,092.19 of tax."
-us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Assumed the 2024 brackets (2.36%/3.15%/3.54%/4.72%/5.12%) still apply in 2026. The rates have since been cut to 2.11%/2.81%/3.16%/4.22%/4.58%, which yields $3,092.19 rather than $3,460."
-us,scenario_014,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"Subtracted about $20,600 of deductions and exemptions when WV allows only $4,000 of personal exemptions. It also used outdated 2.36%–5.12% rates instead of the 2026 schedule of 2.11%–4.58%."
-us,scenario_014,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Carried the 2.27%/3.02%/3.40%/4.53%/4.92% schedule into 2026. It missed the subsequent cuts to 2.11%/2.81%/3.16%/4.22%/4.58%, so it reported $3,322.46 instead of $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Subtracted the ESI premium from AGI and then subtracted the federal standard deduction, which WV does not allow; WV allows only $4,000 of exemptions. It also used invented 2%/3%/4%/5% rates instead of the 2026 schedule (2.11%–4.58%)."
-us,scenario_014,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Had the right ~$84,900 base but applied rates near the 2024 schedule. The 2026 rates (2.11%/2.81%/3.16%/4.22%/4.58%) give $3,092.19, not $3,422."
-us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"Deducted the $14,717 ESI premium and an invented $8,000 WV standard deduction, which gave $62,211 of taxable income instead of $84,927.65. It also applied a nonexistent flat 4.5% rate instead of the graduated 2026 brackets."
-us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Used the 2024 brackets ($2,183.50 + 5.12% above $60k) on the correct $84,928 base. The 2026 schedule tops out at 4.58% and totals $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Claimed the personal exemption reduces taxable wages to zero. The WV exemption is only $2,000 per person ($4,000 total), which leaves $84,927.65 taxable and $3,092.19 of tax."
-us,scenario_014,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"Applied a blended 2.36%–4.82% schedule to the correct $84,928 base. The 2026 rates are 2.11%/2.81%/3.16%/4.22%/4.58%, giving $3,092.19, not $3,265."
-us,scenario_014,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"Taxed $84,928 at the 2025 brackets with a 4.82% top rate instead of the 2026 brackets with a 4.58% top rate (2.11%/2.81%/3.16%/4.22%/4.58%). That overstates the tax by $163."
-us,scenario_014,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Applied the 2025 top rate of 4.82% with inconsistent bracket amounts (438, 552, 854) instead of the 2026 schedule (2.11%/2.81%/3.16%/4.22%/4.58%). That schedule gives 211 + 421.50 + 474 + 844 + 1,141.69 = $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Subtracted the ESI premium, which put taxable income at $70,211, and applied an inverted schedule with rates falling from 4.80% to 2.88%. It also took a nonexistent 4% family tax credit, although WV rates rise from 2.11% to 4.58% and no credit applies."
-us,scenario_014,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Applied the federal standard deduction, which WV does not allow, and assumed nonrefundable credits eliminate the tax. WV allows only $4,000 of exemptions, which leaves $84,927.65 taxable and $3,092.19 owed."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It had the right $84,928 base but used the 2025 schedule (2.22%–4.82%), which gives about $3,256. It then raised that to $3,374 with no stated basis. The 2026 schedule is 2.11%/2.81%/3.16%/4.22%/4.58%, which gives $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It had the right $84,928 base but guessed 2026 rates of 2.13/2.84/3.20/4.26/4.63%. The engine's 2026 rates are 2.11/2.81/3.16/4.22/4.58%, so each bracket is slightly overstated and the result is about $33 too high."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,thresholds_rates,False,"It explicitly carried the 2025 schedule (2.22%–4.82%) into 2026 and got $3,255.03. The 2026 trigger cut lowered the rates to 2.11%–4.58%, which gives $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $14,717 ESI premium from wages, but the engine's AGI is the full $88,927.65. It also skipped the $4,000 in WV personal exemptions. Instead of the 2026 graduated brackets, it applied an invented ~4.5% average rate."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It gave a rough $2,700 with no bracket computation. Taxing $84,927.65 (wages less two $2,000 exemptions) under the 2026 brackets of 2.11%–4.58% gives $3,092.19. Its figure implies an effective rate of only about 3.2% on taxable income."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It computed $3,255 using the 2025 schedule instead of the 2026 schedule (2.11%–4.58%). It then knocked off about $378 for unnamed nonrefundable credits, even though no WV nonrefundable credit applies to this household."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It mixed rates that don't belong together (4.07% and 4.625% alongside 2025 lower brackets) to get $3,130. It then applied an invented extra rate cut to reach $2,849 instead of using the 2026 brackets of 2.11/2.81/3.16/4.22/4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It had the right $84,928 base but its $1,655 implies an effective rate of only about 1.95%. That means it never actually ran the graduated brackets. The 2026 brackets give $3,092.19, and no nonrefundable credits apply."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It used the 2025 schedule (2.22%–4.82%) on the right $84,928 base. It missed the 2026 reduction to 2.11%/2.81%/3.16%/4.22%/4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used an outdated four-bracket structure with a 4.72% top rate to get $3,360. It then arbitrarily cut that to $1,919, citing credits it had itself said don't apply. The 2026 five-bracket schedule (2.11%–4.58%) on $84,928 gives $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It had the right $84,928 base but never applied a specific schedule and rounded to $3,200 on a 4.82% top-rate assumption. The 2026 brackets top out at 4.58%, which gives $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,It applied the 2024 schedule (2.36%–5.12%) to the right base. That misses both the 2025 cut and the 2026 cut that brought the rates down to 2.11%–4.58%.
+us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It invented a $4,000 WV standard deduction and $6,000-per-person exemptions. WV has no standard deduction and allows only $2,000 per exemption, so its taxable income of $72,928 is too low. It also used a made-up rate schedule instead of 2.11%–4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It cut AGI to $74,211 by subtracting the ESI premium and applied a non-existent $15,400 WV standard deduction. It never took the actual $4,000 in personal exemptions, so its taxable income was $58,811 instead of $84,927.65. Its rate schedule was also invented."
+us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $74,211 by subtracting the ESI premium and added a fictitious $4,000 standard deduction on top of the exemptions. That gave $66,211 instead of $84,927.65 of taxable income, even though its rates were near the 2026 schedule."
+us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted $8,000 instead of the $4,000 allowed (two $2,000 exemptions, no standard deduction). It also used rates (1.69%–4.29%) well below the 2026 schedule of 2.11%–4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It started from a $74,211 AGI by subtracting the $14,717 ESI premium, but the engine's AGI is the full $88,927.65 in wages. It then taxed the understated $70,211 base at the 2024 rates (2.36%–5.12%) instead of the 2026 rates (2.11%–4.58%)."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no computation. $4,125 matches roughly a 4.6–4.9% flat rate on the whole wage or taxable amount, ignoring the graduated brackets. The 2026 brackets of 2.11%–4.58% on $84,927.65 give $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used a $74,211 AGI by subtracting the ESI premium, so its taxable income was $70,211 instead of $84,927.65. Its $2,706 also matches the 2024 rates (2.36%–5.12%) rather than the 2026 schedule."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $14,717 ESI premium to get a $74,211 AGI and $70,211 of taxable income. The engine taxes $84,927.65. Its $2,708 also matches the outdated 2024 rate schedule."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It claimed deductions and exemptions eliminate the tax. WV allows only $4,000 in exemptions and no standard deduction, which leaves $84,927.65 taxable and $3,092.19 of tax."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $74,211 by subtracting the ESI premium, giving $70,211 of taxable income instead of $84,927.65. It applied the 2024 rates (2.36%–5.12%, which gives exactly $2,706.30) instead of the 2026 rates of 2.11%–4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It took a $74,211 AGI by subtracting the ESI premium and taxed only $70,211 instead of $84,927.65. It then used an unstated schedule that is heavier than WV's actual 2026 rates."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It had the right $84,928 base but applied the 2024 rates (2.36/3.15/3.54/4.72/5.12%). It missed the cuts that bring 2026 down to 2.11/2.81/3.16/4.22/4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"It had the right $84,928 taxable income, but its $3,459.81 is the 2024 schedule (2.36%–5.12%). The 2026 rates of 2.11%–4.58% give $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It invented an $8,000 WV standard deduction (WV has none) and then applied the 4.82% top rate to all of its taxable income instead of using graduated brackets. The 2026 rates are also 2.11%–4.58%, not a top rate of 4.82%."
+us,scenario_014,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It computed Kentucky tax on a made-up $25,050 income and applied a family size tax credit. The household is in West Virginia, where $84,927.65 of taxable income produces $3,092.19 of tax."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no derivation. $2,331 implies an effective rate of about 2.7% on taxable income, well below what the 2026 brackets (2.11%–4.58%) give on $84,927.65, which is $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and nonrefundable credits wipe out the liability. WV allows only $4,000 in exemptions and no nonrefundable credit applies to this childless, mid-income couple, so $3,092.19 is owed."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the 2025 joint brackets (2.22%–4.82%) to the right $84,928 base. It missed the 2026 reduction to 2.11/2.81/3.16/4.22/4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It had the right $84,928 base but used a schedule only about 1% below the 2025 rates. The 2026 schedule (2.11%–4.58%) is about 5% below 2025 and gives $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $3,255.03 is the 2025 schedule (2.22%–4.82%) applied to $84,928. It missed the 2026 rates of 2.11%–4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $3,255 matches the 2025 WV schedule on the right base. It missed the further 2026 rate reduction to 2.11%–4.58%, which gives $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,It explicitly used the 2025 rates of 2.22/2.96/3.33/4.44/4.82%. The 2026 schedule is 2.11/2.81/3.16/4.22/4.58%.
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It had the right $84,928 base but used an unspecified schedule about 2% below 2025. The 2026 schedule is about 5% below 2025 (2.11%–4.58%) and gives $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It applied the 2025 rates (2.22%–4.82%) to $84,928. It missed the 2026 trigger cut to 2.11%–4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"It had the right $84,928 base but overshot the 2026 rate cut, using rates about 9% below 2025. The engine's 2026 schedule of 2.11%–4.58% gives $3,092.19, not $2,952.20."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"Its $3,255.03 is the 2025 schedule on the right base. It missed the 2026 rates of 2.11/2.81/3.16/4.22/4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It assumed a standard deduction zeroes out the tax. WV has no standard deduction, so $84,927.65 remains taxable after the $4,000 in exemptions, producing $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,It explicitly assumed the 2024 brackets (2.36%–5.12%) still apply in 2026. The 2025 and 2026 cuts brought them down to 2.11%–4.58%.
+us,scenario_014,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted about $20,600 in deductions and exemptions, but WV allows only $4,000 (two $2,000 exemptions) and no standard deduction. It also used the outdated 2.36%–5.12% rates."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It had the right $84,928 base but used rates of 2.27/3.02/3.40/4.53/4.92% carried over from 2025. The 2026 schedule is 2.11/2.81/3.16/4.22/4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premium from AGI and then took the federal standard deduction, which WV does not allow. WV allows only $2,000 per exemption. It also used invented 2%/3%/4%/5% rates instead of 2.11%–4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It had the right ~$84,900 base, but its $3,422 implies near-2024 rates. The 2026 schedule (2.11%–4.58%) gives $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premium to get a $74,211 AGI and invented an $8,000 WV standard deduction. It then applied a non-existent 4.5% flat rate instead of WV's graduated 2026 brackets of 2.11%–4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It applied the 2024 schedule ($2,183.50 on the first $60,000 plus 5.12% above). The 2026 schedule is 2.11%–4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated the WV personal exemption as large enough to eliminate taxable wages. The exemption is only $2,000 per person ($4,000 total), which leaves $84,927.65 taxable and $3,092.19 owed."
+us,scenario_014,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It had the right $84,928 base but blended the 2024 bottom rate (2.36%) with the 2025 top rate (4.82%). It never used the 2026 rates of 2.11/2.81/3.16/4.22/4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It taxed $84,928 under the 2025 brackets topping out at 4.82%. The 2026 top rate is 4.58%, with the lower brackets cut to match."
+us,scenario_014,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used the 2025 top rate of 4.82% and botched its own middle-bracket amounts ($438, $552, $854). It never used the 2026 schedule of 2.11/2.81/3.16/4.22/4.58%."
+us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It subtracted the ESI premium to get a $74,211 AGI and used an invented, inverted schedule where rates fall from 4.80% to 2.88% as income rises. It also applied a family tax credit that doesn't apply at this income."
+us,scenario_014,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied the federal standard deduction and assumed nonrefundable credits eliminate the tax. WV allows neither, so $84,927.65 stays taxable and produces $3,092.19."
us,scenario_015,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It correctly derived AGI of $42,909, taxable income of $26,809, and tax of about $2,969 using the $12,400 10% bracket. It then dropped that result and submitted $3,103, a figure with no supporting calculation."
us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 standard deduction ($14,600) and 10% bracket ($11,600) instead of the 2026 values ($16,100 / $12,400). It also counted nontaxable workers' comp and deducted the full $4,510 capital loss instead of the $3,000 cap. It then raised its own $3,094 result to $3,509 without any calculation."
us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave no figures and cited 'preferential capital gain treatment', which doesn't apply because net capital is a $4,510 loss limited to a $3,000 ordinary deduction. The correct path is $26,809 taxable income taxed at 10%/12%, or $2,969. Its rounded $2,900 matches no actual bracket calculation."
@@ -1309,66 +1323,68 @@ us,scenario_015,head_medicare_eligible,claude-fable-5,llm_error,age_disability,F
us,scenario_015,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model decided that being disabled qualifies an adult under 65 for Medicare. That skips the requirement of 24 months of Social Security Disability Insurance entitlement. The 36-year-old head has no SSDI income and no ESRD or ALS status, so they are not Medicare eligible."
us,scenario_015,head_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model based eligibility only on the head being listed as disabled. Disability-based Medicare requires 24 months of SSDI benefit receipt, and a disability flag alone doesn't meet that. The head is 36 and gets no Social Security disability benefits, so the answer is not eligible (0)."
us,scenario_015,head_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,False,"The model treated the head's disabled status as enough for Medicare. Under-65 eligibility requires 24 months of SSDI entitlement, or ESRD or ALS status. The head is 36 and has none of these, so Medicare eligibility is 0."
-us,scenario_015,local_income_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"Took the prompt's four-city list as exhaustive and left out Indiana county income tax, which PolicyEngine counts in local_income_tax. The default Adams County rate of 1.6% on Indiana AGI of $41,909.38 gives $670.55, not $0."
-us,scenario_015,local_income_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"Said outright that Indiana county taxes fall outside the output, reading the NYC/Philadelphia/KC/St. Louis list as exhaustive. PolicyEngine puts the county tax in local_income_tax: 1.6% (Adams County) × $41,909.38 Indiana AGI = $670.55."
-us,scenario_015,local_income_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"Wrongly claimed Indiana has no local income tax. Every Indiana county levies a county income tax on Indiana AGI, and PolicyEngine includes it in local_income_tax. The default Adams County rate of 1.6% on $41,909.38 gives $670.55."
-us,scenario_015,local_income_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Ruled Indiana county income tax out of scope because the prompt lists only four cities. PolicyEngine's local_income_tax includes it: 1.6% Adams County rate on Indiana AGI of $41,909.38 (federal AGI $42,909.38 minus the $1,000 exemption) = $670.55."
-us,scenario_015,local_income_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"Read the four-city list as the full scope of the output and never applied Indiana county income tax. PolicyEngine applies the default Adams County 1.6% rate to Indiana AGI of $41,909.38, giving $670.55."
-us,scenario_015,local_income_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"Said the output covers only the four listed city taxes, so it left out Indiana county income tax. PolicyEngine includes that tax in local_income_tax: 1.6% × $41,909.38 Indiana AGI = $670.55."
-us,scenario_015,local_income_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Expressly put Indiana county income tax outside the output category, based on the four-city list. PolicyEngine's local_income_tax includes it, at Adams County's 1.6% on Indiana AGI of $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"Applied only the four listed city jurisdictions and left out Indiana county income tax. PolicyEngine counts that tax in local_income_tax and computes 1.6% (Adams County) × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"Checked only for NYC, Philadelphia, Kansas City and St. Louis residence and never considered Indiana county income tax. PolicyEngine includes that tax in this output: 1.6% on Indiana AGI of $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"Treated the special city taxes as the only things in the output and missed Indiana county income tax. PolicyEngine applies the default Adams County 1.6% rate to $41,909.38 of Indiana AGI, giving $670.55."
-us,scenario_015,local_income_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"Concluded Indiana is not a covered jurisdiction because it isn't in the four-city list. PolicyEngine's local_income_tax includes Indiana county tax: 1.6% × $41,909.38 Indiana AGI = $670.55."
-us,scenario_015,local_income_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"Checked only the four enumerated cities and left out Indiana county income tax. PolicyEngine computes that tax at Adams County's 1.6% on Indiana AGI of $41,909.38 for $670.55."
-us,scenario_015,local_income_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Limited the output to the four listed city jurisdictions and never applied Indiana county income tax. PolicyEngine includes it: 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"Read the four-city list as exhaustive and missed Indiana county income tax, which PolicyEngine puts in local_income_tax. The default Adams County 1.6% rate on Indiana AGI of $41,909.38 gives $670.55."
-us,scenario_015,local_income_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"Wrongly claimed no local income tax applies in Indiana. All Indiana counties levy an income tax on Indiana AGI, and PolicyEngine computes it at Adams County's 1.6% on $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"Knew Indiana has county income taxes but excluded them because the prompt lists only four cities. PolicyEngine includes county tax in local_income_tax: 1.6% × $41,909.38 Indiana AGI = $670.55."
-us,scenario_015,local_income_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"Said no local income tax applies and never accounted for Indiana county income tax. PolicyEngine levies that tax at the default Adams County 1.6% rate on Indiana AGI of $41,909.38, for $670.55."
-us,scenario_015,local_income_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Limited the output to the four listed cities and left out Indiana county income tax. PolicyEngine includes it: 1.6% (Adams County) × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"Checked only the NYC, Philadelphia, Kansas City and St. Louis taxes and missed Indiana county income tax. PolicyEngine includes that tax in local_income_tax and computes $670.55 on Indiana AGI of $41,909.38."
-us,scenario_015,local_income_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"Wrongly said Indiana has no local income taxes. Indiana counties tax Indiana AGI, and PolicyEngine applies the default Adams County 1.6% rate to $41,909.38 for $670.55."
-us,scenario_015,local_income_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"Took the four-city list as the 'standard' local jurisdictions and left out Indiana county income tax. PolicyEngine's local_income_tax includes it: 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Asserted that only the four cities are modeled in this output, which drops Indiana county income tax. PolicyEngine computes that tax at Adams County's 1.6% on Indiana AGI of $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"Read the output set as only the four listed cities and never applied Indiana county income tax. PolicyEngine includes that tax: 1.6% × $41,909.38 Indiana AGI = $670.55."
-us,scenario_015,local_income_tax,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"Looked only for qualifying city taxes and left out Indiana county income tax, which PolicyEngine puts in local_income_tax. The default Adams County 1.6% rate on $41,909.38 gives $670.55."
-us,scenario_015,local_income_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"Read the four enumerated city taxes as the full scope and missed Indiana county income tax. PolicyEngine computes it at 1.6% on Indiana AGI of $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"Concluded Indiana has none of the listed taxes and stopped there, never applying Indiana county income tax. PolicyEngine includes that tax in this output: 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Checked only the four listed jurisdictions and left out Indiana county income tax. PolicyEngine applies Adams County's 1.6% rate to Indiana AGI of $41,909.38, for $670.55."
-us,scenario_015,local_income_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"Took the missing county as meaning no local tax. PolicyEngine defaults the Indiana household to Adams County and applies its 1.6% county income tax to Indiana AGI of $41,909.38, giving $670.55."
-us,scenario_015,local_income_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"Limited the output to the four listed city taxes and never applied Indiana county income tax. PolicyEngine includes it: 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"Excluded Indiana because it is not among the four specified jurisdictions and no county facts were given. PolicyEngine defaults to Adams County and computes county income tax of 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"Assumed only the NYC, Philadelphia, KC and St. Louis taxes are modeled in this output and missed Indiana county income tax. PolicyEngine includes it at 1.6% on Indiana AGI of $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Read the listed taxes as exhaustive and left out Indiana county income tax. PolicyEngine's local_income_tax applies the Adams County 1.6% rate to $41,909.38 of Indiana AGI, giving $670.55."
-us,scenario_015,local_income_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"Checked only residence in the covered cities and never applied Indiana county income tax, which PolicyEngine includes in this output. The tax is 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"Limited the output to the four named city taxes and missed Indiana county income tax. PolicyEngine computes $670.55 from Adams County's 1.6% rate on Indiana AGI of $41,909.38."
-us,scenario_015,local_income_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"Considered only NYC, Philadelphia, Kansas City and St. Louis residence and left out Indiana county income tax. PolicyEngine includes it: 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"Read the output as covering only the listed cities and never applied Indiana county income tax. PolicyEngine levies it at Adams County's 1.6% on Indiana AGI of $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,gpt-6.1-terra,prompt_ambiguity,state_local_rule,False,"Expressly put Indiana local income taxes outside the specified output. PolicyEngine includes Indiana county income tax in local_income_tax: 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"Wrongly claimed Indiana has no applicable local income tax. Indiana counties tax Indiana AGI, and PolicyEngine applies the default Adams County 1.6% rate to $41,909.38 for $670.55."
-us,scenario_015,local_income_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"Checked only the four enumerated city taxes and missed Indiana county income tax. PolicyEngine computes it at 1.6% on Indiana AGI of $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"Limited the output to the four listed cities and never applied Indiana county income tax. PolicyEngine includes it in local_income_tax: 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"Read the four-city list as exhaustive and left out Indiana county income tax. PolicyEngine applies Adams County's 1.6% rate to Indiana AGI of $41,909.38, giving $670.55."
-us,scenario_015,local_income_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"Limited the scope to the four specified categories and missed Indiana county income tax. PolicyEngine includes it: 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,inkling,prompt_ambiguity,state_local_rule,False,"Considered only the four listed city taxes and left out Indiana county income tax, which PolicyEngine counts in local_income_tax. The 1.6% Adams County rate on $41,909.38 gives $670.55."
-us,scenario_015,local_income_tax,kimi-k2.6,prompt_ambiguity,state_local_rule,False,"Took the enumerated NYC/Philadelphia/KC/St. Louis taxes as the full scope and never applied Indiana county income tax. PolicyEngine computes it at 1.6% on Indiana AGI of $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"Expressly put Indiana county taxes outside the output's scope. PolicyEngine includes them in local_income_tax and applies Adams County's 1.6% rate to $41,909.38 for $670.55."
-us,scenario_015,local_income_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"Assumed PolicyEngine's local-income-tax output covers only the four cities and left out Indiana county income tax. The engine includes that tax: 1.6% × $41,909.38 Indiana AGI = $670.55."
-us,scenario_015,local_income_tax,muse-spark-1.3,prompt_ambiguity,state_local_rule,False,"Checked only for the four listed city taxes and missed Indiana county income tax. PolicyEngine computes it at Adams County's 1.6% on Indiana AGI of $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"Limited the output to the listed city taxes and never applied Indiana county income tax. PolicyEngine includes it: 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"Excluded Indiana county income tax from the output and treated the missing county as meaning no tax. PolicyEngine includes county tax in local_income_tax, defaults to Adams County, and computes 1.6% × $41,909.38 = $670.55."
-us,scenario_015,local_income_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"Read the listed jurisdictions as exhaustive and left out Indiana county income tax. PolicyEngine applies the Adams County 1.6% rate to Indiana AGI of $41,909.38, giving $670.55."
-us,scenario_015,local_income_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"Checked only the four enumerated city taxes and missed Indiana county income tax, which PolicyEngine counts in local_income_tax. The tax is 1.6% × $41,909.38 = $670.55."
-us,scenario_015,payroll_tax,deepseek-v4.1-flash,llm_error,payroll_tax_base,False,"It subtracted the $2,080 health insurance premiums from wages as a pre-tax cafeteria-plan exclusion and applied 7.65% to $42,920. The FICA base here is the full $45,000 of gross wages, since the listed premiums are not a Section 125 salary reduction. The correct total is $2,790 + $652.50 = $3,442.50."
-us,scenario_015,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It lowered FICA wages to $42,920 by treating the $2,080 health insurance premiums as a pre-tax payroll deduction. Employee Social Security and Medicare tax apply to the full $45,000 of gross wages, which gives $3,442.50 rather than $3,283."
-us,scenario_015,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It assumed the $2,080 employer-sponsored insurance premiums were pre-tax and cut FICA wages to $42,920. The 6.2% and 1.45% employee rates apply to the full $45,000 of gross wages, giving $2,790 + $652.50 = $3,442.50."
-us,scenario_015,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It defined FICA taxable wages as $45,000 minus the $2,080 health premiums, treating those premiums as a Section 125 exclusion. The correct base is the full $45,000 of gross wages, so the 7.65% employee FICA total is $3,442.50, not $3,283.38."
-us,scenario_015,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It correctly computed Social Security of $2,790 and Medicare of $652.50 on $45,000 of wages, but reported their sum as $3,445.50 instead of $3,442.50. The error is purely in the final addition."
-us,scenario_015,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It found the correct components, Social Security of $2,790 and Medicare of $653 (rounded from $652.50), but summed them to $3,419 instead of about $3,443. The correct total is $3,442.50, so the miss is an addition error."
-us,scenario_015,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no parseable value and no explanation for payroll_tax. The correct derivation is 7.65% of $45,000 in wages: $2,790 Social Security plus $652.50 Medicare, or $3,442.50."
-us,scenario_015,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It correctly reached $3,442.50 from $2,790 Social Security and $652.50 Medicare, then raised it to $3,495 for made-up 'Indiana county tax adjustments'. Indiana county taxes are local income taxes, not employee payroll taxes, and Indiana has no mandatory employee state payroll tax. Nothing should be added to the FICA total."
-us,scenario_015,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"It subtracted the $1,042 traditional 401(k) deferral from the FICA base. Under IRC 3121(v)(1)(A), elective 401(k) deferrals remain subject to Social Security and Medicare tax, so the base stays at the full $45,000. It then submitted $3,425.67, which does not match its own computed total of $3,362.79. The correct amount is $3,442.50."
+us,scenario_015,local_income_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"Read the four cities in the prompt's definition as the full list and returned 0. PolicyEngine's local_income_tax includes Indiana county income tax: the default Adams County rate of 1.6% on Indiana AGI of $41,909.38 gives $670.55."
+us,scenario_015,local_income_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"Left Indiana county income tax out of the output on purpose because the prompt listed only NYC, Philadelphia, Kansas City, and St. Louis. PolicyEngine includes the county tax: 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55."
+us,scenario_015,local_income_tax,claude-fable-5.2,prompt_ambiguity,state_local_rule,False,"Treated the four named cities as the only taxes in scope, so an Indiana resident got 0. PolicyEngine adds Indiana county income tax using the default Adams County rate of 1.6% on $41,909.38 Indiana AGI, which is $670.55."
+us,scenario_015,local_income_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"Wrongly said Indiana has no separate local income tax. Every Indiana county levies a local income tax on Indiana AGI. With the default Adams County rate of 1.6% on $41,909.38 ($42,909.38 federal AGI minus the $1,000 exemption), the tax is $670.55."
+us,scenario_015,local_income_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Knew about Indiana county income tax but ruled it out of scope based on the prompt's four-city list. PolicyEngine includes it: 1.6% Adams County default × $41,909.38 Indiana AGI = $670.55."
+us,scenario_015,local_income_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"Read the four-city definition as the full scope and returned 0. PolicyEngine's local_income_tax also covers Indiana county income tax, which comes to $670.55 at the default Adams County 1.6% rate on Indiana AGI of $41,909.38."
+us,scenario_015,local_income_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"Treated the output as covering only the NYC, Philadelphia, Kansas City, and St. Louis taxes. The engine also includes Indiana county income tax and defaults to Adams County when no county is given: 1.6% × $41,909.38 = $670.55."
+us,scenario_015,local_income_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Explicitly left out Indiana county income tax because of the four-city list. PolicyEngine counts it in local_income_tax, computing $670.55 from the Adams County 1.6% rate on Indiana AGI of $41,909.38."
+us,scenario_015,local_income_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"Limited the output to the listed city jurisdictions and returned 0. PolicyEngine includes Indiana county income tax: the default Adams County rate of 1.6% on $41,909.38 Indiana AGI gives $670.55."
+us,scenario_015,local_income_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"Concluded that no local tax applies because the household is not in one of the four named cities. PolicyEngine also counts Indiana county income tax, which is $670.55 at Adams County's 1.6% rate on Indiana AGI of $41,909.38."
+us,scenario_015,local_income_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"Treated the four named city taxes as the whole output. The engine adds Indiana county income tax with Adams County as the default county: 1.6% × ($42,909.38 AGI − $1,000 exemption) = $670.55."
+us,scenario_015,local_income_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"Said Indiana is outside the covered jurisdictions because only four cities were listed. PolicyEngine's local_income_tax includes Indiana county income tax, which comes to $670.55 at the default Adams County 1.6% rate."
+us,scenario_015,local_income_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"Returned 0 because the household is not in NYC, Philadelphia, Kansas City, or St. Louis. PolicyEngine also applies Indiana county income tax: 1.6% Adams County default on $41,909.38 Indiana AGI = $670.55."
+us,scenario_015,local_income_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Treated the four named city taxes as the only ones in the output. PolicyEngine includes Indiana county income tax, which gives $670.55 at the default Adams County rate of 1.6% on Indiana AGI of $41,909.38."
+us,scenario_015,local_income_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"Considered only the listed city earnings taxes and returned 0. The engine adds Indiana county income tax: 1.6% (Adams County default) × $41,909.38 = $670.55."
+us,scenario_015,local_income_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"Wrongly said no local income tax applies in Indiana. Indiana counties levy local income taxes on Indiana AGI, and the default Adams County rate of 1.6% on $41,909.38 gives $670.55."
+us,scenario_015,local_income_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"Recognized that Indiana has county income taxes but left them out because the definition named only four cities. PolicyEngine includes them in local_income_tax: Adams County default 1.6% × $41,909.38 = $670.55."
+us,scenario_015,local_income_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"Returned 0 by applying the prompt's four-city scope. PolicyEngine computes Indiana county income tax of $670.55 at the default Adams County rate of 1.6% on Indiana AGI of $41,909.38."
+us,scenario_015,local_income_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Read the four named cities as the full scope and excluded an Indiana resident. The engine includes Indiana county income tax: 1.6% Adams County default on $41,909.38 = $670.55."
+us,scenario_015,local_income_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"Checked only the NYC, Philadelphia, Kansas City, and St. Louis taxes. PolicyEngine also applies Indiana county income tax, which is $670.55 at Adams County's 1.6% rate on Indiana AGI of $41,909.38."
+us,scenario_015,local_income_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"Wrongly said Indiana has no local income taxes. All 92 Indiana counties levy one, and the default Adams County rate of 1.6% on Indiana AGI of $41,909.38 gives $670.55."
+us,scenario_015,local_income_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"Limited the output to the four named city jurisdictions. PolicyEngine includes Indiana county income tax: 1.6% (Adams County default) × $41,909.38 Indiana AGI = $670.55."
+us,scenario_015,local_income_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Assumed only the four listed cities are modeled. PolicyEngine also models Indiana county income tax and returns $670.55 at the default Adams County 1.6% rate on $41,909.38 Indiana AGI."
+us,scenario_015,local_income_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"Treated the four-city list as the full output set and returned 0. The engine counts Indiana county income tax: 1.6% × ($42,909.38 AGI − $1,000 exemption) = $670.55."
+us,scenario_015,local_income_tax,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"Looked only for qualifying city taxes and did not include Indiana county income tax. PolicyEngine's local_income_tax includes it, computing $670.55 at the default Adams County rate of 1.6%."
+us,scenario_015,local_income_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"Treated the four named local taxes as the full scope. PolicyEngine adds Indiana county income tax using Adams County as the default: 1.6% × $41,909.38 Indiana AGI = $670.55."
+us,scenario_015,local_income_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"Returned 0 because Indiana has none of the four listed city taxes. PolicyEngine's output also covers Indiana county income tax, which is $670.55 at the Adams County 1.6% rate on $41,909.38."
+us,scenario_015,local_income_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Relied on the four-city list and found no matching jurisdiction. The engine includes Indiana county income tax with a default county: 1.6% Adams County rate × $41,909.38 = $670.55."
+us,scenario_015,local_income_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"Returned 0 because no locality was given. PolicyEngine defaults an Indiana household to Adams County and applies its 1.6% county income tax to Indiana AGI of $41,909.38, which gives $670.55."
+us,scenario_015,local_income_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"Treated the four named city taxes as the full output. PolicyEngine includes Indiana county income tax: 1.6% Adams County default on $41,909.38 Indiana AGI = $670.55."
+us,scenario_015,local_income_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"Excluded Indiana because it is not among the specified jurisdictions and no county was given. The engine defaults to Adams County and applies its 1.6% county income tax to Indiana AGI of $41,909.38, which gives $670.55."
+us,scenario_015,local_income_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"Assumed only the NYC, Philadelphia, Kansas City, and St. Louis taxes are modeled separately. PolicyEngine's local_income_tax also includes Indiana county income tax, which is $670.55 at the default Adams County 1.6% rate."
+us,scenario_015,local_income_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Read the prompt's list of local taxes as complete and returned 0. PolicyEngine includes Indiana county income tax: 1.6% × $41,909.38 Indiana AGI = $670.55."
+us,scenario_015,local_income_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"Limited the output to residents of the four covered cities. The engine also computes Indiana county income tax at the default Adams County rate of 1.6% on $41,909.38 Indiana AGI, which is $670.55."
+us,scenario_015,local_income_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"Required residence in one of the four named cities and returned 0. PolicyEngine adds Indiana county income tax: Adams County default rate of 1.6% × $41,909.38 = $670.55."
+us,scenario_015,local_income_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"Treated the four named cities as the only taxes in scope. PolicyEngine includes Indiana county income tax, which comes to $670.55 at the default Adams County 1.6% rate on Indiana AGI of $41,909.38."
+us,scenario_015,local_income_tax,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"Excluded Indiana as outside the cities covered by the output. The engine counts Indiana county income tax: 1.6% (Adams County default) × ($42,909.38 AGI − $1,000 exemption) = $670.55."
+us,scenario_015,local_income_tax,gpt-6.1-terra,prompt_ambiguity,state_local_rule,False,"Explicitly put Indiana local income taxes outside the output's scope. PolicyEngine includes them in local_income_tax, computing $670.55 at Adams County's 1.6% rate on $41,909.38."
+us,scenario_015,local_income_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"Wrongly said Indiana has no applicable local income taxes. Every Indiana county levies a county income tax on Indiana AGI, and the default Adams County rate of 1.6% on $41,909.38 gives $670.55."
+us,scenario_015,local_income_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"Checked only the four named city taxes and returned 0. PolicyEngine includes Indiana county income tax: 1.6% Adams County default × $41,909.38 Indiana AGI = $670.55."
+us,scenario_015,local_income_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"Treated non-residence in the four listed cities as meaning zero local tax. The engine adds Indiana county income tax of $670.55 at the default Adams County 1.6% rate on $41,909.38."
+us,scenario_015,local_income_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"Limited the output to NYC, Philadelphia, Kansas City, and St. Louis. PolicyEngine's local_income_tax also covers Indiana county income tax: 1.6% × $41,909.38 Indiana AGI = $670.55."
+us,scenario_015,local_income_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"Applied only the four named categories and returned 0. PolicyEngine includes Indiana county income tax, which is $670.55 at the Adams County default rate of 1.6% on Indiana AGI of $41,909.38."
+us,scenario_015,local_income_tax,inkling,prompt_ambiguity,state_local_rule,False,"Treated the four named city taxes as the full output scope. The engine includes Indiana county income tax with Adams County as the default: 1.6% × $41,909.38 = $670.55."
+us,scenario_015,local_income_tax,kimi-k2.6,prompt_ambiguity,state_local_rule,False,"Concluded that none of the four specific local taxes apply and returned 0. PolicyEngine also applies Indiana county income tax, which is $670.55 at the default Adams County 1.6% rate on Indiana AGI of $41,909.38."
+us,scenario_015,local_income_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"Explicitly put Indiana county taxes outside the output's scope. PolicyEngine includes them in local_income_tax: 1.6% Adams County default × $41,909.38 = $670.55."
+us,scenario_015,local_income_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"Assumed the output covers only the NYC, Philadelphia, Kansas City, and St. Louis taxes. The engine includes Indiana county income tax and returns $670.55 at the Adams County 1.6% rate on $41,909.38 Indiana AGI."
+us,scenario_015,local_income_tax,muse-spark-1.3,prompt_ambiguity,state_local_rule,False,"Looked only for the four named city taxes. PolicyEngine includes Indiana county income tax: the default Adams County rate of 1.6% on Indiana AGI of $41,909.38 gives $670.55."
+us,scenario_015,local_income_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"Treated the listed city taxes as the only components of the output. PolicyEngine adds Indiana county income tax of $670.55: 1.6% × ($42,909.38 AGI − $1,000 exemption)."
+us,scenario_015,local_income_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"Excluded Indiana county income tax on scope grounds and noted that no county was given. PolicyEngine includes the tax and defaults to Adams County, applying 1.6% to Indiana AGI of $41,909.38 for $670.55."
+us,scenario_015,local_income_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"Restricted the output to the four listed jurisdictions and returned 0. The engine includes Indiana county income tax: 1.6% Adams County default × $41,909.38 = $670.55."
+us,scenario_015,local_income_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"Returned 0 because none of the four named city taxes apply. PolicyEngine's local_income_tax also covers Indiana county income tax, which is $670.55 at Adams County's 1.6% rate on Indiana AGI of $41,909.38."
+us,scenario_015,payroll_tax,claude-fable-5.2,llm_error,payroll_tax_base,False,"The model subtracted the $1,042 traditional 401(k) deferral from the FICA wage base and used $43,958. Elective 401(k) deferrals stay subject to Social Security and Medicare tax under IRC §3121(v)(1)(A), so the base is the full $45,000, which gives $3,442.50."
+us,scenario_015,payroll_tax,deepseek-v4.1-flash,llm_error,payroll_tax_base,False,"The model treated the $2,080 of health insurance premiums as pre-tax Section 125 deductions and lowered FICA wages to $42,920. The household facts list no cafeteria-plan salary reduction, so payroll tax applies to the full $45,000 of gross wages, which gives $3,442.50."
+us,scenario_015,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the $2,080 of health premiums from gross wages as pre-tax FICA exclusions and applied 7.65% to $42,920. The premiums do not reduce the payroll tax base here, so 7.65% applies to $45,000, which gives $3,442.50."
+us,scenario_015,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model assumed the employer-sponsored insurance premiums of $2,080 were pre-tax and cut FICA wages to $42,920. Social Security (6.2%) and Medicare (1.45%) apply to the full $45,000 of wages, giving $2,790 + $652.50 = $3,442.50."
+us,scenario_015,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model reduced FICA taxable wages by the $2,080 of health premiums to get $42,920. No Section 125 exclusion applies, so the 7.65% employee rate applies to the full $45,000, giving $3,442.50."
+us,scenario_015,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model used the right method, with $2,790 of Social Security and $652.50 of Medicare on $45,000 of wages. It then submitted $3,445.50 instead of the correct sum of $3,442.50, a $3 addition or transcription error."
+us,scenario_015,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly identified Social Security of $2,790 and Medicare of about $653 on $45,000 of wages, but reported a total of $3,419. The components sum to $3,442.50, so the submitted figure is an arithmetic error."
+us,scenario_015,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no usable payroll_tax value or explanation, so no answer could be scored. The correct derivation is 7.65% of the $45,000 in wages, which gives $3,442.50."
+us,scenario_015,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model correctly computed $2,790 + $652.50 = $3,442.50, then inflated it to $3,495 for invented 'Indiana county tax adjustments.' Indiana county taxes are local income taxes, not employee payroll taxes, and Indiana has no mandatory employee state payroll tax, so nothing should be added."
+us,scenario_015,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model subtracted the $1,042 traditional 401(k) deferral from FICA wages and computed $3,362.79, even though elective deferrals remain subject to FICA. It then submitted $3,425.67, a number that matches neither its own computation nor the correct $3,442.50 on the full $45,000 of wages."
us,scenario_015,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It correctly computed $41,909 × 2.95% ≈ $1,236. It then cut the answer to $1,234 for an unspecified 'rounding in exemption computation', even though the $1,000 exemption is exact and needs no rounding."
us,scenario_015,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It built AGI incorrectly in two ways. It included the $960 of workers' compensation, which is excludable, and it deducted the full $4,510 capital loss instead of the $3,000 cap. It also skipped the $1,000 exemption, applied a 3.15% rate instead of 2.95%, and then lowered the result by an invented disability adjustment."
us,scenario_015,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It used 3.05% instead of the 2026 rate of 2.95%. Its $1,000 answer matches no computation, because even its own rate on the $41,909 base gives about $1,278. The correct result is $41,909.38 × 2.95% = $1,236.33."
@@ -1433,48 +1449,49 @@ us,scenario_016,ssi,glm-5.3,parse_contract_failure,missing_output,False,All wron
us,scenario_016,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_016,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_016,tanf,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"AGI, the $16,100 standard deduction and the brackets were all right, but it also subtracted a $1,625 car-loan interest deduction. That deduction requires a new, U.S.-assembled vehicle bought with a loan taken out after 2024, and none of those facts are given. Dropping it gives taxable income of $45,492 and tax of $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It made up a $6,760 overtime deduction (10 hrs × $13 × 52) and took a $1,625 car-loan interest deduction. No overtime premium was listed, and the engine's overtime deduction counts only a listed FLSA overtime premium. The vehicle also isn't shown to qualify, so both deductions are $0 and taxable income is $45,492.09, not $37,107."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100. It then applied an invented bracket setup (12% on the first $46,225 with no 10% bracket) to get $5,716, and finally submitted an unrelated $4,903."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted health premiums from wages, but they are not a listed pre-tax payroll deduction; the only above-the-line items are the 401(k) and IRA contributions. It then gave a rounded $4,700 guess instead of applying 10%/12% to taxable income of $45,492.09."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its own steps correctly reached taxable income of $45,492 and tax of $1,240 + $3,971 = $5,211. It then dropped that result for an unexplained 'rounded bracket' adjustment and submitted $6,754."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly found taxable income of $45,492 and the bracket pieces $1,240 + $3,971. It added them up wrong, submitting $6,033 instead of $5,211."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It assumed the job is FLSA non-exempt and took a $6,760 overtime deduction, and also took a $1,625 car-loan interest deduction. No overtime premium is listed (the engine's overtime deduction counts only a listed FLSA overtime premium), and the vehicle isn't shown to qualify, so both are $0 and taxable income is $45,492.09."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 pre-OBBBA standard deduction of $15,000 and the 2025 10% bracket edge of $11,925, instead of the 2026 values of $16,100 and $12,400. It also took a $200 Saver's Credit, but single filers lose that credit above $40,250 of AGI in 2026, and this filer's AGI is $61,592."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used $12,150 for the top of the 10% bracket instead of $12,400 and got $5,216. It then subtracted about $303 for no stated reason, even though it had just said the Saver's Credit was phased out at this income."
-us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It itemized after counting the $9,000 premium twice ($18,000 total) and including $1,500 of over-the-counter costs, which are not deductible medical expenses. The correct figure is ($9,000 + $500 − 7.5% of AGI) plus Arizona tax, about $6,000, well below the $16,100 standard deduction. It also used a $15,400 standard deduction and a $12,075 10% bracket edge."
-us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using a $7,700 standard deduction, a $4,950 personal exemption and a 15% bracket. The OBBBA made the TCJA structure permanent: a $16,100 standard deduction, no exemption, and 10%/12% brackets. It also took the $9,000 premium out of wages, which isn't a listed pre-tax deduction."
-us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $15,650 standard deduction instead of the 2026 OBBBA amount of $16,100, and $12,450 instead of $12,400 for the 10% bracket edge. That overstated taxable income by $450 and gave $5,264.04."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied a TCJA expiry ($8,300 standard deduction, $5,300 personal exemption, 15%/25% brackets) that the OBBBA cancelled. The 2026 rules are a $16,100 standard deduction, no exemption, and 10%/12% brackets, giving taxable income of $45,492.09."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used the 2024 standard deduction of $14,600 instead of $16,100 and left out both the $289 IRA deduction and the $50 of interest, reaching taxable income of $47,231. Even on that figure, $4,724 is too low for 10%/12% brackets."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used projected TCJA-expiry brackets and a combined standard deduction plus personal exemption of $13,790. In 2026 the OBBBA rules apply: a $16,100 standard deduction, no exemption, and 10% up to $12,400 with 12% above."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expired ($8,300 standard deduction plus a $5,300 exemption, 15% bracket) instead of the OBBBA $16,100 standard deduction with 10%/12% brackets. It also took the $9,000 health premium out of wages, though the premium isn't a listed pre-tax deduction."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no working. Its $5,304 is about $93 above the correct $5,211.05, which fits a standard deduction below $16,100 or 2025 bracket edges applied to AGI of $61,592."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $15,700 standard deduction instead of the 2026 OBBBA amount of $16,100, and $12,275 instead of $12,400 for the 10% bracket edge. That overstated taxable income at $45,892."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied a TCJA expiry (standard deduction plus personal exemption, 10%/15% brackets) that the OBBBA cancelled. The 2026 rules are a $16,100 standard deduction and 10%/12% brackets, giving taxable income of $45,492.09 and tax of $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 pre-OBBBA standard deduction of $15,000 and the 2025 10% bracket edge of $11,925, instead of the 2026 values of $16,100 and $12,400."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It subtracted a $6,760 overtime deduction and a $1,625 car-loan interest deduction. No overtime premium is listed (the engine's overtime deduction counts only a listed FLSA overtime premium), and the vehicle isn't shown to qualify, so both are $0 and taxable income is $45,492.09."
-us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction instead of the 2026 OBBBA amount of $16,100, and $12,260 instead of $12,400 for the 10% bracket edge. That overstated taxable income at $46,192."
-us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used the 2025 pre-OBBBA standard deduction of $15,000 instead of $16,100, so taxable income came out $1,100 too high at $46,592."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It subtracted a nonrefundable child tax credit for a child who doesn't exist. The household is one single adult with no dependents, so no credits apply and the tax is $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It assumed there was no tax liability, but AGI of $61,592 minus the $16,100 standard deduction leaves $45,492.09 of taxable income. With no credits available, that produces $5,211.05 of tax."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It subtracted a $6,760 overtime deduction based only on 50 usual weekly hours. No overtime premium is listed, and the engine's overtime deduction counts only a listed FLSA overtime premium, so the deduction is $0 and taxable income is $45,492.09, not $38,732."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Taxable income of $45,492 was correct, but it used the 2025 10% bracket edge of $11,925 instead of the 2026 edge of $12,400. Taxing those extra $475 at 12% instead of 10% adds $9.50, giving $5,220.54."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It took an 'estimated' $6,760 overtime deduction (10 hrs × $13 × 52). No overtime premium is listed, and the engine's overtime deduction counts only a listed FLSA overtime premium, so the deduction is $0 and taxable income is $45,492.09."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It subtracted a $6,760 overtime deduction that the facts don't support; no overtime premium is listed, and the engine's overtime deduction counts only a listed FLSA overtime premium. With that removed, taxable income is $45,492.09 and tax is $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It took a $6,760 overtime deduction, but no overtime premium is listed and the engine's overtime deduction counts only a listed FLSA overtime premium. The deduction is $0, so taxable income is $45,492.09, not $38,732."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It mixed up AGI (about $61k) with taxable income and gave a rounded $5,200 guess. It never applied 10% up to $12,400 and 12% above to the actual taxable income of $45,492.09."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied a TCJA expiry ($5,300 personal exemption, 10%/15% brackets) that the OBBBA cancelled. It also itemized after counting the $9,000 premium twice. Deductible medical costs are only ($9,000 + $500) minus 7.5% of AGI, so the $16,100 standard deduction is larger and should have been used."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction instead of the 2026 OBBBA amount of $16,100, and $12,250 instead of $12,400 for the 10% bracket edge. That overstated taxable income at $46,192."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The $16,100 standard deduction and $45,492 taxable income were correct, but it put the top of the 10% bracket at $12,200 instead of $12,400. Taxing those $200 at 12% adds $4, giving $5,215.04."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $15,375 standard deduction instead of the 2026 OBBBA amount of $16,100, so taxable income came out $725 too high at $46,217."
-us,scenario_018,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized a medical deduction of about $15,381, which requires counting the $9,000 premium twice and including over-the-counter costs. Allowable medical costs are $9,500 minus 7.5% of AGI, about $4,881. Even with state tax added, that is far below the $16,100 standard deduction, which should have been used."
-us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score against the $5,211.05 reference."
-us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It submitted $0 with no working. A single filer with $61,592 of AGI and no credits has $45,492.09 of taxable income after the $16,100 standard deduction, which means $5,211.05 of tax."
-us,scenario_018,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It took a $6,760 overtime deduction that isn't supported, since no overtime premium is listed and the engine's overtime deduction counts only a listed FLSA overtime premium. It also used a $15,750 standard deduction and the 2025 $11,925 bracket edge instead of the 2026 values of $16,100 and $12,400."
-us,scenario_018,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It subtracted a $6,760 overtime deduction and a $1,625 car-loan interest deduction. No overtime premium is listed, and nothing shows a new, U.S.-assembled vehicle financed after 2024, so both are $0 and taxable income is $45,492.09."
-us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It left out the $289 traditional IRA deduction (getting AGI of $61,881) and used a $15,750 standard deduction instead of the 2026 amount of $16,100. It also put the 10% bracket edge at $12,550 instead of $12,400."
-us,scenario_018,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"Its taxable income of $32,942 is about $12,550 too low, because it subtracted deductions beyond the $16,100 standard deduction that the facts don't support. It also subtracted a $303 Saver's Credit, but AGI of $61,592 is above the 2026 single-filer limit of $40,250, so the credit is $0."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"AGI ($61,592) and the $16,100 standard deduction are right, but the model also took the $1,625 car loan interest deduction. That deduction requires interest on a loan for a new, US-assembled vehicle, and no such facts are given. Dropping it gives taxable income of $45,492 and tax of $5,211, not $5,016."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The model took a $6,760 overtime deduction (based on a $13 half-time premium) and the $1,625 car loan interest deduction. Neither applies: the $68,005 wage total equals straight time for 50 hours a week ($26 × 50 × 52 ≈ $67,600), so there is no overtime premium, and no qualifying vehicle loan is listed. Taxable income should be $45,492, not $37,107."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"The model made up a $6,182 overtime premium (wages × 5/55) and also deducted $1,625 of car loan interest. The reference allows neither, because the wages show no time-and-a-half pay and no qualifying new-vehicle loan is listed. Only the $16,100 standard deduction applies, giving $45,492 taxable and $5,211 of tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used the 2024 standard deduction ($14,600) instead of the 2026 amount ($16,100). It then taxed income from $0 at 12% instead of applying the 10% bracket up to $12,400. Its reasoning reaches $5,716, yet it submitted $4,903, which matches none of its own steps."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model subtracted health premiums from wages, but the facts never say they are paid pre-tax and the reference does not exclude them. It then guessed about $4,700 on 'roughly $50,000' of taxable income without running the brackets. The correct result is $45,492 taxable income taxed at 10%/12%, or $5,211."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's reasoning got exactly the right answer: $1,240 + 12% × $33,092 = $5,211. It then dropped that result for an unexplained 'adjustment with rounded brackets' and submitted $6,754."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model set up the correct calculation, $1,240 + $3,971 on $45,492 of taxable income, which totals $5,211. It then reported $6,033, an addition error not supported by any bracket step."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model assumed FLSA non-exempt status to claim a $6,760 overtime deduction and assumed the car loan qualified for a $1,625 deduction. Status flags that aren't listed count as false, and the wage total shows no overtime premium, so neither deduction applies. Taxable income should be $45,492 and tax $5,211."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the 2025 single standard deduction ($15,000) and 10% bracket top ($11,925) instead of the 2026 figures ($16,100 and $12,400). It also took a $200 Saver's Credit using AGI limits that are not the single-filer limits. The single limit is about $40,000, so AGI of $61,592 gets no credit."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used a 10% bracket ending at $12,150 instead of $12,400. It then subtracted about $303 from its own $5,216 for no stated reason, even though it said the Saver's Credit was phased out. With no credits, tax on $45,492 is $5,211."
+us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model counted the $9,000 premium twice, adding the 'excluding Medicare Part B' total to the 'other premiums' line, which is the same $9,000. That pushed itemized deductions above the standard deduction. Counted once, medical plus state tax comes to about $7,500, so the $16,100 standard deduction applies. The model also used the wrong standard deduction ($15,400) and 10% bracket ($12,075)."
+us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model assumed the TCJA expired in 2026 ($7,700 standard deduction, personal exemption, 15% bracket), but OBBBA made the TCJA structure permanent with a $16,100 single standard deduction. It also removed $9,000 of health premiums from wages that the facts don't show as pre-tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"AGI of $61,592 is right, but the model used a $15,650 standard deduction and a 10% bracket to $12,450. The 2026 figures are $16,100 and $12,400. That overstated taxable income by $450, giving $5,264 instead of $5,211."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expired, bringing back an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. OBBBA kept the TCJA rates permanent, with a 2026 single standard deduction of $16,100 and 10%/12% brackets that put $45,492 of taxable income at $5,211."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used the 2024 standard deduction of $14,600 and left out the $289 IRA deduction and the $50 of interest. Its stated $4,724 doesn't even follow from its own $47,231 taxable income. The correct result is $45,492 taxable and $5,211 of tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied pre-TCJA brackets and a personal exemption on the assumption that the TCJA had expired. OBBBA made the $16,100 standard deduction and 10%/12% brackets permanent for 2026, so tax is $5,211 on $45,492 of taxable income."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed the TCJA expired ($8,300 standard deduction plus $5,300 exemption, 15% bracket) instead of applying OBBBA's permanent $16,100 standard deduction. It also subtracted $9,000 of health premiums from wages even though the facts don't show them as pre-tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no working. $5,304 matches a taxable income of about $46,200, which means a standard deduction of roughly $15,400 instead of the 2026 amount of $16,100. The correct $45,492 taxable income produces $5,211."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"AGI is right, but the model used a $15,700 standard deduction and a 10% bracket to $12,275. The 2026 values are $16,100 and $12,400. That overstated taxable income by $400 and gave $5,261.54."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied the TCJA-expired law (personal exemption, small standard deduction, 15% bracket) to 2026. OBBBA made the TCJA permanent, so the $16,100 standard deduction and 10%/12% brackets apply, giving $5,211."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model used the 2025 single standard deduction ($15,000) and 10% bracket top ($11,925) instead of the 2026 figures ($16,100 and $12,400). That gave $46,592 taxable and $5,352.54 instead of $45,492 and $5,211."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"The model subtracted a $6,760 overtime deduction and a $1,625 car loan interest deduction. No overtime premium is reported ($68,005 matches straight time at $26 × 50 hours × 52 weeks), and no qualifying new-vehicle loan is listed. Only the $16,100 standard deduction applies."
+us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a $15,400 standard deduction and a 10% bracket to $12,260 instead of the 2026 values of $16,100 and $12,400. That gave $46,192 of taxable income and $5,297.84 of tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model used the 2025 standard deduction of $15,000 instead of the 2026 amount of $16,100, and a 10% bracket to about $12,300 instead of $12,400. That overstated taxable income at $46,592 and gave $5,345."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model applied a nonrefundable Child Tax Credit for a qualifying child, but the household is one single adult with no dependents. With no credits, tax on $45,492 is $5,211."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model assumed no positive tax liability without running the calculation. AGI of $61,592 minus the $16,100 standard deduction leaves $45,492 taxable, which is $5,211 of tax with no nonrefundable credits to offset it."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model added a $6,760 qualified overtime deduction on top of the $16,100 standard deduction. The wage total ($26 × 50 × 52 ≈ $68,005) contains no time-and-a-half premium, so no overtime deduction applies. Taxable income is $45,492, not $38,732."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"AGI, the standard deduction, and taxable income ($45,492) are all right. However, $5,220.54 only results from ending the 10% bracket at the 2025 figure of $11,925 instead of the 2026 figure of $12,400, which adds $9.49 of tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model deducted an estimated $6,760 overtime premium (10 hours × $13 × 52). The reported wages equal straight-time pay for 50 hours, so there is no qualified overtime compensation. Taxable income is $45,492 and tax $5,211."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model applied a $6,760 qualified overtime deduction, but no overtime premium exists in the $68,005 of wages ($26 × 50 × 52). Only the $16,100 standard deduction applies, giving $45,492 taxable and $5,211 of tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"The model deducted an overtime premium, and its $4,399.84 matches the $6,760 overtime deduction. The wages show no time-and-a-half pay, so that deduction doesn't apply. Taxable income is $45,492 after only the standard deduction, giving $5,211."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model called about $61k 'taxable income', but that is the AGI, and it gave a rounded $5,200 without working through the brackets. Subtracting the $16,100 standard deduction leaves $45,492, which works out to exactly $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model assumed the TCJA expired (personal exemption, 15% bracket), but OBBBA made it permanent. It also itemized $18,500 of medical expenses by counting the same $9,000 premium twice. Counted once, itemized deductions fall well below the $16,100 standard deduction."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used a $15,400 standard deduction and a 10% bracket to $12,250 instead of the 2026 values of $16,100 and $12,400. That overstated taxable income at $46,192 and gave $5,298."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Taxable income of $45,492 is right, but the model ended the 10% bracket at $12,200 instead of the 2026 figure of $12,400. That taxed $200 at 12% instead of 10% and gave $5,215.04."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $15,375 standard deduction instead of the 2026 single amount of $16,100. That left $46,217 of taxable income instead of $45,492 and produced $5,302."
+us,scenario_018,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model's medical deduction of about $15,381 comes from $20,000 of expenses, which counts the $9,000 premium twice (the 'excluding Part B' total plus the identical 'other premiums' line). With the premium counted once, itemized deductions are well below $16,100, so the standard deduction applies."
+us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for this output, so there was no answer to score."
+us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model submitted $0 with no derivation. The correct path is AGI of $61,592, minus the $16,100 standard deduction, for $45,492 taxable at 10%/12%, which is $5,211. A zero answer means the model treated the filer as having no tax liability."
+us,scenario_018,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"The model subtracted a $6,760 overtime premium deduction that doesn't apply, since the wages show no time-and-a-half pay. It also used a $15,750 standard deduction and a $11,925 10% bracket instead of the 2026 values of $16,100 and $12,400."
+us,scenario_018,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"The model deducted a $6,760 overtime premium and $1,625 of car loan interest. No overtime premium is reported in the wages, and no qualifying new-vehicle loan is listed, so neither applies. Taxable income is $45,492, not $37,107."
+us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model left out the $289 traditional IRA deduction, putting AGI at $61,881 instead of $61,592. It also used a $15,750 standard deduction and a 10% bracket to $12,550 instead of the 2026 values of $16,100 and $12,400."
+us,scenario_018,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model subtracted about $12,550 of unexplained extra deductions to reach $32,942 of taxable income, when the correct figure is $45,492. It also claimed a $303 Saver's Credit even though AGI of $61,592 exceeds the single-filer limit of about $40,000."
us,scenario_018,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_018,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly computed $4,216.31 of Social Security tax and $986.07 of Medicare tax, which sum exactly to $5,202.38, then invented a $49.41 “rounding adjustment.” No such adjustment applies."
us,scenario_018,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the listed $9,000 health-insurance premiums from FICA wages as though they were documented pre-tax employer-plan salary reductions. Applying 7.65% to the full $68,005 wage base yields $5,202.38."
@@ -1522,57 +1539,58 @@ us,scenario_018,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_er
us,scenario_018,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It wrongly treated the $68,005 gross wages as already net of the $6,174 traditional 401(k) contribution. It also dropped the $289 IRA deduction, so AGI came out at $68,055 instead of $61,592. On top of that it used a $14,600 deduction instead of $16,100."
us,scenario_018,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Its taxable income of $45,486 is close to correct, even though it used a $14,508 deduction and a lower implied AGI. It then applied an invented 'rate structure' of about 1.83%, giving $832. Arizona's flat 2.5% rate gives about $1,137."
us,scenario_018,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,other,False,"Its AGI of $300,150 and its NIIT were correct, but it counted only the $13,834 of real estate taxes as SALT. The standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 amount of $1,595 leaves SALT below $16,100. It left out the $196 non-itemizer charitable deduction, so its taxable income of $284,050 is $195.81 above the convention's $283,854.19, and its $68,404 is $69.27 above the $68,334.73 that policyengine-us 2.60.0 gives with NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,other,False,"It took the $16,100 standard deduction plus the $196 non-itemizer charitable deduction, which totals $16,296. That is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below the standard deduction; only the frozen reference's projected sales tax makes itemized SALT $17,090. Its $68,334.59 is $0.14 below the $68,334.73 that policyengine-us 2.60.0 gives with NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,other,False,"It subtracted the $138,375 of unreimbursed employee business expenses above the line, but these are permanently non-deductible after OBBBA, and it used a $15,000 standard deduction. It then dropped its own $27,268 result for an unsupported $66,661. It never took the 2026 $16,100 standard deduction plus the $196 non-itemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice, and never added NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,other,False,"It said liability was high but submitted 0 without doing the calculation. Under the release's sales tax convention (c_irs_sales_tax_2025), AGI of $300,150 less the $16,100 standard deduction and the $196 non-itemizer charitable deduction gives taxable income of $283,854.19. Tax is $68,334.73 with NIIT, or the exclusion record's $68,112.09 without it."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,other,False,"It added the $42,857 FLSA overtime premium on top of the $300,000 of wages, although the prompt says gross wages already include overtime. That inflated AGI to $343,007. It also capped SALT at $10,000 instead of the 2026 $40,400 cap and left out the sales tax deduction. Its reported $70,813 does not match its own $82,743 calculation."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,other,False,"It disallowed the $5,709 alimony deduction, which the reference takes above the line. It also made up $23,855 of mortgage interest from the balance, although unlisted interest counts as 0, and capped SALT at $10,000 without adding sales tax. Its final $71,866 does not match its own roughly $64,291 calculation."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,other,False,"It guessed itemized deductions of $30,000–40,000 and gave an unsupported $72,700. Itemized deductions are SALT alone, and the release's sales tax convention (c_irs_sales_tax_2025) leaves them below the $16,100 standard deduction. On a $300,150 AGI, the standard deduction plus the $196 non-itemizer charitable deduction yields $68,334.73 with NIIT, or $68,112.09 without it."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,other,False,"It took the $16,100 standard deduction plus the $196 non-itemizer charitable deduction. That is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below the standard deduction. Its $68,334.59 is $0.14 below the $68,334.73 that policyengine-us 2.60.0 gives with NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,other,False,"It treated alimony as non-deductible, giving AGI of $305,859 instead of $300,150, and capped SALT at $10,000 without the sales tax deduction. It used a self-projected $15,400 standard deduction and brackets instead of the OBBBA 2026 figures ($16,100; 35% bracket starting at $256,225). It also left out the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,other,False,"It used the 2025 $15,000 standard deduction and 2025 brackets instead of 2026 values. It capped SALT at $10,000 without sales tax, and added $900 of Additional Medicare Tax, which is a payroll tax and not income tax. It then rounded its $70,467 subtotal up to an unsupported $76,500."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,other,False,"It treated the $5,709 alimony payment as non-deductible, so AGI was $305,859 instead of $300,150. It also took the $16,100 standard deduction without the $196 non-itemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025). Both errors overstated taxable income."
-us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,other,False,"It assumed the TCJA expired in 2026. On that basis it deducted unreimbursed employee expenses above a 2% floor, claimed a personal exemption, and used pre-TCJA brackets. OBBBA made the TCJA rules permanent, so miscellaneous itemized deductions and personal exemptions are still eliminated, and taxable income is $283,854.19 under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 non-itemizer charitable deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,other,False,"It applied pre-TCJA rules: a $132,426 miscellaneous deduction for employee business expenses, a personal exemption, and 15%/25%/28% brackets. OBBBA made the TCJA permanent, so none of these apply. It also subtracted the $8,389 of employer health premiums from the stated gross wages and skipped the alimony deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,other,False,"It claimed a $25,000 overtime deduction. The single cap is $12,500, and it phases out by $100 per $1,000 of MAGI over $150,000, so it is zero at $275,000 of MAGI. It also skipped the alimony deduction and used a $15,350 standard deduction instead of the $16,100 standard deduction plus the $196 non-itemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice. It also left out NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,other,False,"It reverted to pre-TCJA law, deducting $132,258 of employee business expenses above the 2% floor and taking a $5,450 personal exemption under 15%/25%/28% brackets. OBBBA made the TCJA permanent, so both deductions are eliminated in 2026. It also skipped the above-the-line alimony deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,other,False,"It assumed TCJA expiry and built a pre-TCJA regular tax with an AMT override. Under permanent TCJA/OBBBA rules, employee business expenses are not deductible and AMT does not apply. It also subtracted the $8,389 of employer health premiums from the stated gross wages and skipped the $5,709 alimony deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,other,False,"It started from a $305,859 AGI without the $5,709 alimony deduction. It capped real estate taxes at $10,000 instead of using the $40,400 2026 SALT cap with the sales tax addition. Its standard deduction is still the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below the $16,100 standard deduction even under the $40,400 cap."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,other,False,"It assumed the TCJA expired, allowed unreimbursed employee expenses as a 2%-floor miscellaneous deduction, and then computed AMT. OBBBA made the TCJA permanent, so those expenses are not deductible and ordinary tax applies on $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 non-itemizer charitable deduction. It also wrongly subtracted the employer health premiums from the stated gross wages."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,other,False,"It cut AGI to $291,761 by subtracting the $8,389 of employer health premiums from the stated gross wages. It itemized $146,570 including the unreimbursed employee expenses, which remain permanently disallowed, and then applied AMT. Under the release's sales tax convention (c_irs_sales_tax_2025), the path is AGI of $300,150 with the $16,100 standard deduction plus the $196 non-itemizer charitable deduction, taxed under ordinary brackets."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,other,False,"It used AGI of $305,859 without the alimony deduction and took the standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice. Its $50,568 figure is about $19,600 below the $70,178.70 that the 2026 single brackets give on $289,759 of taxable income, that AGI less the $16,100 standard deduction. It also left out NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,other,False,"It assumed TCJA expiry: uncapped real estate taxes, employee business expenses above a 2% floor, and personal exemptions. All are unavailable under the permanent OBBBA rules. It also subtracted the employer health premiums from the stated gross wages, giving AGI of $291,761 instead of $300,150."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,other,False,"It applied TCJA sunset rules, itemizing unreimbursed employee expenses above 2% of AGI and claiming a personal exemption. Both are permanently eliminated under OBBBA. It also subtracted the $8,389 of employer health premiums from the stated wages and skipped the alimony deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,other,False,"It left out the $5,709 alimony deduction, giving AGI of $305,859, and used the 2025 $15,000 standard deduction. Under the release's sales tax convention (c_irs_sales_tax_2025), the household's deduction is the 2026 $16,100 standard deduction plus the $196 non-itemizer charitable deduction. It also left out the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,other,False,"It computed tax on a $305,859 AGI, so it skipped the above-the-line alimony deduction that brings AGI to $300,150. It took the $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice, but not the $196 non-itemizer charitable deduction. Together these put its taxable income of $289,759 about $5,905 above the convention's $283,854.19."
-us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,other,False,"It got the alimony deduction and AGI of $300,150 right. It then assumed the TCJA expires in 2026 and deducted $132,372 of unreimbursed employee expenses plus a personal exemption under pre-TCJA brackets. OBBBA made the suspension of miscellaneous deductions and personal exemptions permanent."
-us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,other,False,"It declared alimony non-deductible, giving AGI of $305,859 instead of $300,150. It compared $14,030 of itemized deductions (real estate taxes plus charity) with the $16,100 standard deduction and took the standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice. It left out the $196 non-itemizer charitable deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,other,False,"It assumed mortgage interest, medical expenses and unreimbursed employee expenses wiped out all tax. Mortgage interest is $0 because no interest was listed, medical costs are under the 7.5% floor, and employee expenses are permanently non-deductible. Itemized deductions are only SALT, which stays below the $16,100 standard deduction under the release's sales tax convention (c_irs_sales_tax_2025). The standard deduction plus the $196 non-itemizer charitable deduction leaves $68,334.73 of tax with NIIT, or $68,112.09 without it."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,other,False,"Its $31,472 matches an itemized calculation that deducts the $138,375 of unreimbursed employee expenses under a TCJA-sunset assumption. That deduction is permanently disallowed. Itemized deductions are SALT alone, which stays below the standard deduction under the release's sales tax convention (c_irs_sales_tax_2025). The $16,100 standard deduction plus the $196 non-itemizer charitable deduction gives $68,334.73 with NIIT, or $68,112.09 without it."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,other,False,"Its AGI of $300,150 and NIIT were correct, and its $16,100 standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below it. It left out the $196 non-itemizer charitable deduction, so its taxable income of $284,050 was $195.81 above the convention's $283,854.19."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,other,False,"It subtracted the $8,389 of employer health premiums from the stated $300,000 gross wages, although PolicyEngine counts the full wage amount in AGI. Its standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), but it left out the $196 non-itemizer charitable deduction and the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,other,False,"It took the standard deduction plus the $196 non-itemizer charitable deduction as the release's sales tax convention (c_irs_sales_tax_2025) does. It left out the $222.64 NIIT, which the frozen reference includes; its $68,112 is $0.09 below the exclusion record's $68,112.09, which leaves the NIIT out."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,other,False,"Its $70,179 matches tax on $289,759 of taxable income. That means it skipped the $5,709 alimony deduction and took the $16,100 standard deduction without the $196 non-itemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025). It also left out the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,other,False,"It used the standard deduction plus the $196 non-itemizer charity deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice. It left out the $222.64 NIIT, which the frozen reference includes; its $68,111.95 is $0.14 below the exclusion record's $68,112.09, which leaves the NIIT out."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,other,False,"Its $70,401 is tax on a $305,859 AGI, so it skipped the $5,709 above-the-line alimony deduction. It also took the $16,100 standard deduction without the $196 non-itemizer charitable deduction; the standard deduction itself is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025)."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,other,False,"It never subtracted the $5,709 alimony deduction, giving AGI of $305,859, and used the $16,100 standard deduction. The standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), but it left out the $196 non-itemizer charitable deduction, so its taxable income of $289,759 was $5,904.81 above the convention's $283,854.19."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,other,False,"It took the standard deduction plus the $196 non-itemizer charitable deduction. That is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below the standard deduction. It left out the $222.64 NIIT, which the frozen reference includes; its $68,111.95 is $0.14 below the exclusion record's $68,112.09, which leaves the NIIT out."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,other,False,"It applied alimony, NIIT and the qualified-dividend rate correctly and its $16,100 standard deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025). Its $68,403.19 matches tax on $284,050 of taxable income, so it left out the $196 non-itemizer charitable deduction, which overstated taxable income by $195.81."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,other,False,"It gave a rough $65,000 guess without working through deductions or brackets. Under the release's sales tax convention (c_irs_sales_tax_2025), the derivation uses AGI of $300,150, the $16,100 standard deduction and the $196 non-itemizer charitable deduction. That gives $283,854.19 of taxable income and $68,112.09 of regular tax, or $68,334.73 with the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,other,False,"It assumed TCJA expiry and deducted $132,372 of miscellaneous employee expenses plus a personal exemption under pre-TCJA brackets scaled by inflation. OBBBA made those suspensions permanent. Under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 non-itemizer charitable deduction, taxable income is $283,854.19; it computed $148,321."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,other,False,"It assumed the TCJA sunset and computed AMT driven by uncapped SALT and employee expense deductions that do not exist under permanent OBBBA rules. It also skipped the $5,709 alimony deduction, using AGI of $305,859."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,other,False,"It took the $16,100 standard deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice, but left out the $196 non-itemizer charitable deduction, for taxable income of $284,050. Its stated total covers only ordinary and qualified-dividend tax and leaves out the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,other,False,"It applied pre-TCJA law with uncapped SALT, a 2%-floor deduction for $138,375 of employee expenses, and 15%/25%/28% brackets. Those rules do not apply under permanent OBBBA law. It also subtracted the employer health premiums from the stated wages and skipped the alimony deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,other,False,"It subtracted the $8,389 of employer health premiums from the stated gross wages, skipped the $5,709 alimony deduction, and treated the $196 gift as an above-the-line deduction. Under the release's sales tax convention (c_irs_sales_tax_2025), the path is AGI of $300,150 with the $16,100 standard deduction plus the $196 gift as the non-itemizer charitable deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there was nothing to score. The frozen reference is $68,056.71; policyengine-us 2.60.0 gives $68,334.73 under the release's conventions, and the exclusion record's alternative, which leaves out the NIIT, is $68,112.09."
-us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,other,False,"It took the $16,100 standard deduction plus the $196 non-itemizer charitable deduction because it saw only $13,834 of itemizable taxes. That is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), whose IRS 2025 table leaves SALT below the standard deduction even with the sales tax added. Its $68,334.59 is $0.14 below the $68,334.73 that policyengine-us 2.60.0 gives with NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,other,False,"It submitted 0 with no reasoning, which treats a single filer with about $300,000 of AGI as owing nothing. Under the release's sales tax convention (c_irs_sales_tax_2025), the $16,100 standard deduction plus the $196 non-itemizer charitable deduction gives $68,334.73 with NIIT, or the exclusion record's $68,112.09 without it."
-us,scenario_020,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,other,False,"It cut AGI to about $297,470 by subtracting the employer health premiums from the stated gross wages and by denying the alimony deduction. It took a $15,750 standard deduction instead of the 2026 $16,100 standard deduction plus the $196 non-itemizer charitable deduction, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice, and it left out NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,other,False,"It subtracted the $8,389 of employer health premiums from the stated $300,000 gross wages and skipped the alimony deduction, giving AGI of about $297,470. Its standard deduction plus the charitable deduction is the household's choice under the release's sales tax convention (c_irs_sales_tax_2025), and it left out the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,other,False,"It treated alimony as non-deductible, giving AGI of $305,859 instead of $300,150. It took the $16,100 standard deduction over $14,030 of itemized deductions, which the release's sales tax convention (c_irs_sales_tax_2025) makes the household's choice, but it left out the $196 non-itemizer charitable deduction and the $222.64 NIIT."
-us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,other,False,"It added the $42,857 overtime premium on top of $300,000 of wages, although overtime is already included in gross wages. It used a $15,750 standard deduction and capped SALT at $10,000. Its reported $68,513.50 does not match its own $85,241.65 calculation."
-us,scenario_020,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,other,False,"It used married-filing-jointly brackets, a qualified business income deduction and a child tax credit for a single filer with no business income and no children. None of these apply. The correct path is single brackets on $283,854.19 of taxable income under the release's sales tax convention (c_irs_sales_tax_2025), which takes the standard deduction plus the $196 non-itemizer charitable deduction, plus NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,other,False,"The model counted only the $13,834 of real estate taxes as SALT and took the $16,100 standard deduction. It never added the Texas general sales tax deduction from the IRS optional table, which raises itemized SALT to $17,090.34. That error overstated taxable income by about $990 ($284,050 vs $283,059.84), which accounts for the roughly $347 overstatement."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,other,False,"The model left out the Texas general sales tax deduction that lifts SALT to $17,090.34, above the standard deduction. It therefore took the $16,100 standard deduction plus the $196 non-itemizer charitable deduction instead of itemizing $17,090.34 (where charity is $0 under the 0.5%-of-AGI floor). The resulting taxable income of $283,854 instead of $283,059.84 explains the extra $278."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,other,False,"The model treated $13,834 of real estate tax as the only itemizable SALT. It never added the sales tax deduction from the IRS optional table for a no-income-tax state like Texas, which brings SALT to $17,090.34 and makes itemizing beat the $16,100 standard deduction. By using standard deduction plus $196 charity it overstated taxable income by $794."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,other,False,"The model subtracted the $138,375 of unreimbursed employee business expenses above the line, but those are suspended miscellaneous deductions. It used a $15,000 standard deduction and computed $27,268, then substituted an unsupported $66,661. It also never itemized SALT including the Texas sales tax ($17,090.34) or added the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,other,False,"The model acknowledged a high federal liability but never computed it and submitted $0. The correct path is AGI $300,150 after the alimony deduction, $17,090.34 of itemized SALT including the sales tax, taxable income $283,059.84, regular tax $67,834.07 and NIIT $222.64."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,other,False,"The model added the $42,857 FLSA overtime premium on top of $300,000 of gross wages, even though it is already included, and capped SALT at the old $10,000 instead of the $40,400 2026 cap. It also missed the Texas sales tax deduction that makes itemizing ($17,090.34) beat the standard deduction, and it left out NIIT. Its submitted $70,813 does not even match its own $82,743 computation."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,other,False,"The model invented $23,855 of mortgage interest from the loan balance, although unlisted interest is 0. It denied the $5,709 alimony deduction, capped SALT at $10,000 instead of $40,400 and left out the sales tax component, and it used outdated brackets. The submitted $71,866 does not match its own ~$64,291 arithmetic."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,other,False,"The model guessed itemized deductions of $30,000–$40,000 and then submitted an unexplained $72,700 instead of computing the actual itemized total. That total is $13,834 of real estate taxes plus the optional-table sales tax, or $17,090.34, giving taxable income of $283,059.84, tax of $67,834.07 and NIIT of $222.64."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,other,False,"The model compared only the $13,834 of property tax against the $16,100 standard deduction and never added the Texas general sales tax deduction, which brings SALT to $17,090.34. Itemizing is therefore better. Using the standard deduction plus $196 non-itemizer charity overstated taxable income by $794 and tax by $278."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,other,False,"The model treated the $5,709 of alimony paid as nondeductible, which put AGI at $305,859 instead of $300,150. It capped SALT at $10,000 instead of $40,400 and left out the sales tax, so it never itemized the $17,090.34. It also used a projected $15,400 standard deduction and never added the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,other,False,"The model used a $10,000 SALT cap, a $15,000 standard deduction and 2025 brackets instead of the $40,400 cap, the sales-tax-inclusive itemized total of $17,090.34 and 2026 brackets. It wrongly added the $900 Additional Medicare Tax, which is a payroll tax, to income tax. It then inflated its own $70,467 to an arbitrary $76,500."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,other,False,"The model ruled the $5,709 alimony payment nondeductible, which left AGI at $305,859 instead of $300,150. It also missed the Texas sales tax deduction that lifts SALT to $17,090.34 over the $16,100 standard deduction. Together these overstated taxable income by about $6,700 ($289,759 vs $283,059.84)."
+us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,other,False,"The model assumed the TCJA had expired. It applied pre-TCJA brackets and a personal exemption and allowed the $138,375 of unreimbursed employee expenses as a 2%-floor miscellaneous deduction. Under 2026 law (OBBBA), miscellaneous deductions stay suspended and personal exemptions stay at zero, so taxable income is $283,059.84, not about $155,500."
+us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,other,False,"The model subtracted the $8,389 of ESI premiums from the stated gross wages and ignored the alimony deduction. It applied expired pre-TCJA rules: a 2%-floor miscellaneous deduction for $138,375 of employee expenses, a personal exemption and old brackets. Those rules are suspended permanently for 2026, so taxable income is $283,059.84, not $145,559."
+us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,other,False,"The model claimed a $25,000 overtime deduction. The single-filer cap is $12,500, and it phases out by $100 per $1,000 of MAGI over $150,000, so it is fully eliminated at $300,150. The model also skipped the alimony deduction, used a $15,350 standard deduction instead of itemizing $17,090.34 of SALT including sales tax, and left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,other,False,"The model applied TCJA-sunset rules: a deduction for unreimbursed employee expenses above the 2% floor, a $5,450 personal exemption and pre-TCJA brackets. For 2026 those deductions remain suspended. It also ignored the alimony deduction and NIIT, which drove taxable income down to $154,121 instead of $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,other,False,"The model excluded the $8,389 of ESI premiums from the stated gross wages and used the pre-TCJA miscellaneous deduction and personal exemption. It then computed an AMT of $71,411. Under 2026 law the employee expenses are not deductible, so the regular tax on $283,059.84 ($67,834.07) applies with no AMT, plus $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,other,False,"The model started from $305,859 without the $5,709 alimony deduction, capped SALT at the old $10,000 and took the standard deduction. It missed the sales-tax-inclusive itemized SALT of $17,090.34 under the $40,400 cap. The $68,630 is not tied to any shown bracket computation."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,other,False,"The model assumed the TCJA individual provisions expire in 2026, so it allowed the 2%-floor miscellaneous deduction for employee expenses and then imposed AMT. It also excluded ESI premiums from wages. Under 2026 law the miscellaneous deduction stays suspended, and the regular tax on $283,059.84 of taxable income plus NIIT gives $68,056.71 with no AMT."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,other,False,"The model subtracted the $8,389 of ESI premiums from gross wages and itemized $146,570, including the suspended unreimbursed employee expenses. It then applied AMT. The correct AGI is $300,150 and the correct itemized total is $17,090.34 (real estate plus sales tax), leaving taxable income of $283,059.84 with no AMT."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,other,False,"The model omitted the alimony deduction, took a generic standard deduction and gave no bracket computation. Its $50,568 is about $17,500 below the tax on roughly $283,000 of taxable income in the 35% bracket. The correct path itemizes $17,090.34 of SALT including sales tax and adds the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,other,False,"The model applied TCJA-sunset rules: uncapped SALT, a miscellaneous deduction above the 2% floor for $138,375 of employee expenses, and personal exemptions. It also netted ESI premiums out of gross wages. For 2026 those miscellaneous deductions remain suspended and SALT is $17,090.34, so taxable income is $283,059.84, not about $140,000."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,other,False,"The model assumed a 2026 TCJA sunset, deducted the unreimbursed employee expenses above the 2% AGI floor and claimed a personal exemption. It also subtracted ESI premiums from the stated gross wages. Under current 2026 law those deductions do not exist, so its $31,473 understates tax by more than half."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,other,False,"The model omitted the $5,709 alimony deduction, used the $15,000 standard deduction instead of itemizing $17,090.34 of SALT including Texas sales tax, and left out the $222.64 NIIT. That produced taxable income of $290,859 instead of $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,other,False,"The model computed tax on $305,859 of AGI without the $5,709 alimony deduction and took the standard deduction. It missed the sales tax deduction that makes itemizing $17,090.34 worthwhile, so taxable income was about $6,700 too high."
+us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,other,False,"The model treated 2026 as a TCJA-expiration year. It deducted $132,372 of unreimbursed employee expenses as miscellaneous itemized deductions, used a personal exemption and pre-TCJA brackets, and left out the $222.64 NIIT. Those deductions remain suspended, so taxable income is $283,059.84, not $148,748."
+us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,other,False,"The model declared the $5,709 of alimony nondeductible, which kept AGI at $305,859, and compared only $14,030 of itemized deductions to the standard deduction. It left out the Texas general sales tax deduction that raises SALT to $17,090.34 and makes itemizing the better choice. Taxable income came out at $289,759 instead of $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,other,False,"The model claimed that mortgage interest, which is unlisted and therefore 0, plus suspended unreimbursed employee expenses wipe out all tax and submitted $0. Only $17,090.34 of SALT is itemizable, which leaves $283,059.84 of taxable income and $68,056.71 of tax."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,other,False,"The model's $31,472 reflects deducting the $138,375 of unreimbursed employee business expenses as itemized deductions. Those deductions are suspended for 2026, so the only itemized deduction is $17,090.34 of SALT, giving taxable income of $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,other,False,"The model took the $16,100 standard deduction (taxable income $284,050) without seeing that real estate taxes plus the IRS-table Texas sales tax total $17,090.34. Because that exceeds the standard deduction, itemizing is better. Missing it overstated taxable income by $990 and tax by about $347."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,other,False,"The model subtracted the $8,389 of ESI premiums from the stated $300,000 of gross wages, which reduced taxable income to about $275,661. It also used the standard deduction instead of itemizing $17,090.34 of SALT including sales tax, and it left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,other,False,"The model used the standard deduction plus the $196 non-itemizer charitable deduction instead of itemizing $17,090.34 of SALT, which includes the Texas optional-table sales tax. It also left out the $222.64 net investment income tax that applies because MAGI exceeds $200,000."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,other,False,"The model's $70,179 is the ordinary and qualified-dividend tax on $289,759 of taxable income. That means it skipped the $5,709 alimony deduction, took the standard deduction instead of itemizing $17,090.34 of SALT including sales tax, and left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,other,False,"The model took the $16,100 standard deduction plus $196 of non-itemizer charity instead of itemizing $17,090.34 of real estate plus sales tax. It also left out the $222.64 net investment income tax on $5,859 of investment income."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,other,False,"The model's $70,401 matches tax on $289,759 of taxable income plus NIIT. It therefore omitted the $5,709 alimony deduction and used the standard deduction instead of itemizing $17,090.34 of SALT that includes the Texas sales tax amount."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,other,False,"The model started from $305,859 of gross income without subtracting the $5,709 alimony deduction. It applied the $16,100 standard deduction and missed that real estate taxes plus the optional-table sales tax ($17,090.34) exceed it. As a result, taxable income was $289,759 instead of $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,other,False,"The model used the standard deduction plus the $196 non-itemizer charity deduction instead of itemizing $17,090.34 of SALT that includes the Texas general sales tax. It also left out the $222.64 net investment income tax."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,other,False,"The model's $68,403.19 corresponds to the $16,100 standard deduction (taxable income $284,050). It missed the Texas optional-table sales tax deduction that brings SALT to $17,090.34 and makes itemizing better, which overstated tax by about $347."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,other,False,"The model gave a rounded $65,000 guess with no bracket computation. The correct derivation is AGI $300,150 after alimony, $17,090.34 of itemized SALT including sales tax, taxable income $283,059.84, tax $67,834.07 and NIIT $222.64."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,other,False,"The model applied pre-TCJA 2026 sunset rules: a $132,372 miscellaneous deduction for employee expenses, a personal exemption and inflated pre-TCJA brackets. Under 2026 law those remain suspended, so taxable income is $283,059.84, not $148,321."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,other,False,"The model assumed the TCJA sunset and routed tax through AMT. It also ignored the $5,709 alimony deduction (AGI $305,859). Under 2026 law the employee expenses are simply nondeductible, regular tax applies on $283,059.84 after itemizing $17,090.34 of SALT, and no AMT arises."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,other,False,"The model used the $16,100 standard deduction instead of itemizing $17,090.34 of real estate plus Texas sales tax, and it left out the $222.64 NIIT. Its $68,268 also does not match the $68,180.55 that its own $284,050 taxable income produces."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,other,False,"The model subtracted ESI premiums from the stated gross wages and ignored the alimony deduction. It applied pre-TCJA rules: uncapped SALT and a 2%-floor miscellaneous deduction for $138,375 of employee expenses. Those expenses are nondeductible in 2026, so taxable income is $283,059.84, not $151,014."
+us,scenario_020,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,other,False,"The model subtracted the $8,389 of ESI premiums from the stated gross wages and omitted the $5,709 alimony deduction. It treated the $196 of charity as a deduction in reaching AGI, when the gift is $0 once the filer itemizes because it falls below the 0.5% floor. It also took the standard deduction instead of itemizing $17,090.34 of SALT including sales tax."
+us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value and no explanation for federal_income_tax_before_refundable_credits.
+us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,other,False,"The model compared only $13,834 of property tax to the $16,100 standard deduction and omitted the Texas general sales tax deduction, which brings SALT to $17,090.34. It therefore used the standard deduction plus $196 charity instead of itemizing, which overstated taxable income by $794."
+us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,other,False,"The model submitted $0 with no computation, even though $300,150 of AGI produces substantial tax. The correct path gives taxable income of $283,059.84 after $17,090.34 of itemized SALT, tax of $67,834.07 and NIIT of $222.64."
+us,scenario_020,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,other,False,"The model netted the $8,389 of ESI premiums out of the stated gross wages and ruled alimony nondeductible, which put AGI at about $297,470 instead of $300,150. It also took the standard deduction instead of itemizing $17,090.34 of SALT including sales tax, and it left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,other,False,"The model subtracted the $8,389 of pre-tax ESI premiums from the stated gross wages instead of deducting the $5,709 of alimony. It used the standard deduction plus $196 charity instead of itemizing $17,090.34 of real estate plus sales tax, and it left out the $222.64 NIIT."
+us,scenario_020,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,other,False,"The model declared alimony nondeductible, which kept AGI at $305,859, and set itemized deductions at $14,030 without the Texas sales tax, which brings SALT to $17,090.34. It also left out the $222.64 NIIT. The result was $70,178.70 on $289,759 of taxable income instead of $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,other,False,"The model added the $42,857 FLSA overtime premium on top of $300,000 of gross wages, which already include it, and capped SALT at $10,000 instead of $40,400. It also omitted the alimony deduction and NIIT. Its submitted $68,513.50 contradicts its own $85,241.65 computation."
+us,scenario_020,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,other,False,"The model applied married-filing-jointly brackets, a QBI deduction and a child tax credit, none of which fit this single filer with no business income and no children. The correct path is single brackets on $283,059.84 of taxable income after itemizing $17,090.34 of SALT, plus $222.64 NIIT."
us,scenario_020,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_020,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"It derived the correct $16,689 ($11,439 + $4,350 + $900) and then dropped it. It cut Medicare wages by the $8,389 ESI premium as if it were a cafeteria-plan deduction, giving $16,491.86, and then submitted $15,200, which matches neither calculation. The stated $300,000 gross wages are the FICA base, and nothing reduces them."
us,scenario_020,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the outdated 2024 Social Security wage base of $168,600 instead of the 2026 base of $184,500. It also added the $42,857 FLSA overtime premium on top of the $300,000, which already includes overtime, and then subtracted the ESI premium. Its submitted $23,119.50 matches none of its own subtotals, which ranged from $15,703 to $16,710."
@@ -1622,57 +1640,58 @@ us,scenario_021,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model
us,scenario_021,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value for the requested output and therefore failed the required submission contract.
us,scenario_021,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $5,508 taxable pension as Missouri taxable income, ignored the applicable deductions and 2026 taxable-income threshold, and applied an unsupported flat 1.9% rate directly to that pension. Missouri taxable income is reduced to zero before the rate calculation, yielding $0 of tax before refundable credits."
us,scenario_021,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It got single status, the phased $3,951 senior deduction and the $910 QBI deduction right, and it limited SALT to the $8,829 property tax because it treated state income tax as unknown; the prompt lists no state income tax withheld or paid. It also skipped the 0.5%-of-AGI charitable floor, so taxable income came to $80,066. The reference's $74,552.59 adds the engine's $6,058.85 formula estimate of California income tax withheld to SALT."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It used qualifying-surviving-spouse status, which requires a dependent child that this household lacks, so it applied joint brackets and the $33,850 joint standard deduction. The return is single and itemized, with 22% tax above $50,400; itemized deductions are $29,735.52 in the reference, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, and $24,832.27 with no state income tax paid. It also took the full $6,000 senior deduction instead of phasing it down by 6% of MAGI above $75,000 to $3,951."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as head of household with no qualifying person, put the full $20,735 of Social Security into income instead of the 85% taxable amount, and subtracted $8,351 of unreimbursed employee expenses above the line, which is not allowed. It never itemized, never took the senior or QBI deductions, and its final $11,316 does not match its own $10,564 bracket calculation."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated taxable Social Security as about $0 and concluded taxable income was near zero. Provisional income above $100,000 makes 85% ($17,625) taxable, AGI is $109,149, and after the reference's $34,596.57 of deductions, the most under any reading of the unlisted state income tax, taxable income is $74,552.59."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a qualifying surviving spouse, which needs a dependent child, and so used joint brackets and the $33,950 joint standard deduction. The correct return is single and itemized, with income above $50,400 taxed at 22%. It also left out the OBBBA senior deduction entirely."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint brackets although there is no dependent child, made up $12,771 of mortgage interest from the loan balance even though no interest paid was given (it counts as 0), and took the full $6,000 senior deduction without the phaseout. Its final $6,919 does not even follow from its own $7,350 calculation."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only property tax and charity ($24,222), with no state income tax in SALT, which the prompt does not list (the reference adds the engine's $6,058.85 estimate of California income tax withheld), and took the full $6,000 senior deduction without the 6% phaseout above $75,000 of MAGI. It then reported $8,547, which is far below the single-schedule tax on its own $78,017 taxable income."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a qualifying surviving spouse, which needs a dependent child, so it used joint brackets, the $33,850 joint standard deduction and a non-itemizer charitable deduction. As a single filer, itemizing beats the $18,150 single standard deduction under every reading of the unlisted state income tax ($29,735.52 in the reference, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, and $24,832.27 with none paid), and income above $50,400 is taxed at 22%."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026, bringing back personal exemptions, 2% miscellaneous deductions and pre-TCJA brackets, even though OBBBA made the TCJA rules permanent. It also used surviving-spouse joint status without a dependent and imputed $13,835 of mortgage interest that was never given. Its final $13,013 does not match any of its own calculations."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint brackets and a $32,600 joint standard deduction although there is no dependent child, so the return should be single and itemized. It held SALT to the old $10,000 cap (the 2026 single cap is $40,400) and skipped the senior and QBI deductions, and its final $6,800 does not match its own roughly $8,700 bracket result."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a surviving spouse with joint brackets and the $33,850 standard deduction although there is no dependent child, so it taxed income above $24,800 at only 12%. It also took the full $6,000 senior deduction instead of phasing it down to $3,951, and it left out the $910 QBI deduction."
-us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint brackets (2025's $23,850 band) and a $32,500 standard deduction although there is no dependent child. It also left out the OBBBA senior deduction, the $910 QBI deduction and any state income tax in SALT, which the prompt does not list (the reference adds the engine's $6,058.85 estimate of California income tax withheld); a single filer here itemizes under every reading of that input."
-us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied rules from a TCJA sunset that did not happen: a $5,300 personal exemption, $6,168 of 2%-floor miscellaneous employee expenses, and 15%/25% brackets. OBBBA keeps exemptions and miscellaneous deductions at zero and keeps the 10/12/22% schedule. It also put its own $2,019 California tax estimate in SALT; the prompt lists no state income tax withheld or paid, and the reference uses the engine's $6,058.85 estimate of California income tax withheld. It also left out the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA expired, so it took a $5,200 personal exemption and 2%-floor employee expenses, both of which stay eliminated under OBBBA, and it put its own $2,135 California tax estimate in SALT, where the reference uses the engine's $6,058.85 estimate of California income tax withheld, an input the prompt does not list. It skipped the senior and QBI deductions, and its submitted $13,047.69 does not match the $12,372 in its own explanation."
-us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only property tax and charity ($24,222), with no state income tax in SALT, which the prompt does not list; the reference adds the engine's $6,058.85 estimate of California income tax withheld. It also skipped the $3,951 phased senior deduction and the $910 QBI deduction, so taxable income came out at $84,927, against $74,552.59 in the reference."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used rules from a TCJA expiration that did not happen: a $5,000 personal exemption, 2%-floor employee expenses and 10/15/25% brackets. OBBBA keeps exemptions and miscellaneous itemized deductions at zero and keeps the 10/12/22% schedule. It also put no state income tax in SALT, which the prompt does not list (the reference adds the engine's $6,058.85 estimate of California income tax withheld), and left out the $3,951 senior deduction."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized mortgage interest, which is not an input (no interest paid was given, so it is 0), and never mentions state income tax in SALT, the 0.5% charitable floor, the phased $3,951 senior deduction or the $910 QBI deduction. Its $11,335 implies taxable income about $1,000 above the reference's $74,552.59, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, an input the prompt does not list."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly used pre-TCJA itemized deductions and personal exemptions. OBBBA made the TCJA rules permanent, so 2026 has no personal exemption, no 2%-floor miscellaneous deductions and the 10/12/22% brackets. It also skipped the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $6,168 of employee expenses above the 2% floor, took a $5,050 personal exemption and used post-TCJA-expiration rates. OBBBA permanently suspended all three, keeping the 10/12/22% brackets. It also put its own $2,172 California tax estimate in SALT, where the reference uses the engine's $6,058.85 estimate of California income tax withheld, an input the prompt does not list, and it left out the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. $13,446 matches itemizing only property tax and charity (about $24,222) and taxing roughly $85,000 at single rates, with no state income tax in SALT and without the $3,951 senior deduction and the $910 QBI deduction. With those deductions and the engine's $6,058.85 estimate of California income tax withheld in SALT, an input the prompt does not list, the reference's taxable income is $74,552.59."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 limit, but OBBBA's 2026 cap is $40,400, under which SALT is deductible in full: $14,887.85 in the reference, which adds the engine's $6,058.85 estimate of California income tax withheld, an input the prompt does not list, to property tax. It also skipped the $3,951 phased senior deduction, so taxable income came out at $82,846, against $74,552.59 in the reference."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only charity and real estate taxes, and the prompt lists no state income tax withheld or paid (the reference adds the engine's $6,058.85 estimate of California income tax withheld to SALT). Its low $9,260 matches taxing about $81,000 of taxable income at joint 10%/12% rates, but a widow with no dependent files single, where income above $50,400 is taxed at 22%."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized unreimbursed employee business expenses, which are a miscellaneous itemized deduction that OBBBA permanently eliminated. It also left out the $3,951 phased senior deduction and the $910 QBI deduction, so its taxable income and tax came out too high."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"It used qualifying-surviving-spouse status although there is no dependent child, and it took a personal exemption even though OBBBA permanently set exemptions to zero. Both moves pushed its tax well below the single-schedule $11,113.57."
-us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It filed single correctly but applied 2025 brackets, itemized only property tax and charity (with no state income tax in SALT, which the prompt does not list), and skipped the OBBBA senior deduction (phased to $3,951). That left taxable income at $84,016.75; the reference's $74,552.59 adds the engine's $6,058.85 estimate of California income tax withheld to SALT."
-us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted $8,351 of unreimbursed employee expenses as an above-the-line adjustment, which is not allowed, and added the tax-exempt pension to AGI. It also subtracted $5,850 of federal withholding that was never given, even though withholding has nothing to do with tax before refundable credits."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It concluded deductions wiped out the tax. In fact AGI is $109,149, including $52,635 of wages and 85% taxable Social Security, and the reference's total deductions of $34,596.57, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, leave $74,552.59 of taxable income, taxed at $11,113.57; the other readings of that unlisted input leave more tax."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no figures. $1,237 matches taxing only about $12,000 of income, which drastically overstates deductions or understates income compared with $109,149 of AGI and the reference's $74,552.59 of taxable income, which the other readings of the unlisted state income tax only raise."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint brackets although there is no dependent child, and it took a personal exemption that OBBBA permanently eliminated. It also left out the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It named itemized, senior and QBI deductions but gave no amounts. $10,247 at single rates implies taxable income of about $70,600, roughly $3,900 below the reference's $74,552.59, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, an input the prompt does not list. The model gives no amounts that locate the difference."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It used the surviving-spouse joint rate schedule and the age-adjusted joint standard deduction although there is no dependent child. The correct return is single and itemized, with income above $50,400 taxed at 22%."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the surviving-spouse (joint) schedule although there is no dependent child, so too much income stayed in the 12% band. It itemized only charity and property tax, with no state income tax in SALT, which the prompt does not list (the reference adds the engine's $6,058.85 estimate of California income tax withheld), and it left out the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse joint brackets and the $33,850 joint standard deduction although there is no dependent child. As a single filer, itemizing beats the $18,150 standard deduction under every reading of the unlisted state income tax ($29,735.52 in the reference, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, and $24,832.27 with none paid), and income above $50,400 is taxed at 22%."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It used the right categories (itemized, phased senior, QBI), but $9,651 at single rates implies taxable income near $67,900. That is about $6,650 below $74,552.59, so its deductions total about $6,650 more than the reference's $34,596.57, whose itemized $29,735.52 is SALT $14,887.85, including the engine's $6,058.85 estimate of California income tax withheld, plus charity $14,847.67 after the 0.5% floor. The model gives no itemized or QBI amounts that locate the difference."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It filed single and phased the senior deduction correctly, and put itemized deductions at about $26,456 without a breakdown; the reference itemizes $29,735.52 by adding the engine's $6,058.85 estimate of California income tax withheld to SALT, an input the prompt does not list. It also left out the $910 QBI deduction on rental income, so taxable income came out at about $78,742."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It used surviving-spouse brackets and the $33,850 joint standard deduction although there is no dependent child. The correct return is single and itemized, with income above $50,400 taxed at 22%."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It applied surviving-spouse joint rates and the joint age-adjusted standard deduction although there is no dependent child, and it took the senior deduction without the 6% phaseout. As a single filer, itemizing wins, and income above $50,400 is taxed at 22%."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used a $30,000 surviving-spouse standard deduction and joint brackets although there is no dependent child. It left out the age add-on, the senior deduction and the QBI deduction, and it never itemized the SALT and charity that exceed the single standard deduction."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-TCJA-sunset rules (a $5,419 personal exemption, $6,168 of 2%-floor employee expenses, pre-TCJA brackets), but OBBBA made the TCJA rules permanent. It also put its own $2,103 California tax estimate in SALT, where the reference uses the engine's $6,058.85 estimate of California income tax withheld, an input the prompt does not list, and it skipped the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It filed single and took the phased senior deduction and QBI deduction, and put its own $1,974 California tax estimate in SALT, for SALT of $10,803. The prompt lists no state income tax withheld or paid; the reference's $14,887.85 SALT adds the engine's $6,058.85 estimate of California income tax withheld. It also skipped the 0.5%-of-AGI charitable floor, so taxable income came out at $78,092, against $74,552.59 in the reference."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It put its own $1,867 California tax estimate in SALT, giving $10,696; the prompt lists no state income tax withheld or paid, and the reference's $14,887.85 SALT adds the engine's $6,058.85 estimate of California income tax withheld. It also set the QBI deduction to $0 instead of 20% of the $4,550 rental income ($910), and it used wrong 2026 single bracket limits ($12,225/$49,675 instead of $12,400/$50,400)."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It put all $20,735 of Social Security into AGI instead of the 85% taxable amount ($17,624.75), deducted 2%-floor employee expenses and used 15%/25% post-TCJA-sunset brackets. OBBBA made the TCJA brackets permanent and the miscellaneous deduction suspension permanent."
-us,scenario_022,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It made up about $901 of rental depreciation that was never given and put its own estimate of about $2,024 of California tax in SALT, giving $10,853; the prompt lists no state income tax withheld or paid, and the reference's $14,887.85 SALT adds the engine's $6,058.85 estimate of California income tax withheld. It also left out the $910 QBI deduction, so taxable income came out at about $78,538, against $74,552.59 in the reference."
-us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no value and no explanation for federal_income_tax_before_refundable_credits.
-us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It put its own $2,700 California tax estimate in SALT, for SALT of $11,529; the prompt lists no state income tax withheld or paid, and the reference's $14,887.85 SALT adds the engine's $6,058.85 estimate of California income tax withheld. It skipped the 0.5%-of-AGI charitable floor and explicitly took no QBI deduction, even though 20% of the $4,550 rental income ($910) is allowed, so taxable income came out at $78,275.67, against $74,552.59 in the reference."
-us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed taxable income was too low to owe tax after the standard deduction. AGI is $109,149, and even the full $34,596.57 of deductions leaves $74,552.59 of taxable income."
-us,scenario_022,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $26,900 for SALT and charity without a breakdown, and taxed about $82,000; the reference itemizes $29,735.52, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, an input the prompt does not list. That means it skipped both the $3,951 phased senior deduction and the $910 QBI deduction, which bring taxable income down to $74,552.59."
-us,scenario_022,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only property tax and charity ($24,222), with no state income tax in SALT, which the prompt does not list, and it left out the $910 QBI deduction. That left taxable income at $80,976; the reference's $74,552.59 adds the engine's $6,058.85 estimate of California income tax withheld to SALT."
-us,scenario_022,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It filed as a surviving spouse with joint rates and the joint standard deduction although there is no dependent child. It took the full $6,000 senior deduction without the 6% phaseout above $75,000 of MAGI and wrongly ruled out the QBI deduction on rental income."
-us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated AGI at $126,773.50 although its own components add to $109,149, used surviving-spouse joint brackets without a dependent, and imputed $12,771 of mortgage interest from the loan balance even though no interest paid was given. It also left out the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It stacked non-deductible items into itemized deductions (unreimbursed employee expenses, mortgage interest that was never given, medical costs below the 7.5% floor) and treated deductions as nonrefundable credits to reach $0. Across the readings of the unlisted state income tax the engine itemizes at most $29,735.52, the reference's figure, whose SALT adds the engine's $6,058.85 estimate of California income tax withheld, and taxable income is at least $74,552.59."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Used the correct single status, senior phaseout and QBI, but limited SALT to the $8,829 property tax, leaving out the roughly $6,059 of CA income tax that brings SALT to $14,888. Its itemized total was $24,222 instead of $29,736, so taxable income came out $5,513 too high ($80,066 vs $74,553), adding about $1,213 of tax at 22%."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Treated the head as a qualifying surviving spouse and used joint brackets and the $33,850 joint standard deduction. With no dependent child, the head must file single, which means 22% brackets above $50,400 and itemizing $29,736. It also took the full $6,000 senior deduction instead of phasing it to $3,951 for AGI over $75,000, and added an invented $2,000 charitable deduction for non-itemizers."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Applied qualifying-surviving-spouse status, with the joint $33,850 standard deduction and joint 10%/12% brackets, even though there is no dependent child, so the head files single. As a single filer, itemized deductions of $29,736 beat the $18,150 standard deduction, taxable income is $74,553, and the top $24,153 falls in the 22% bracket."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Used head-of-household status without a qualifying person and subtracted the $8,351 of unreimbursed employee business expenses above the line, though they are not deductible. It also counted 100% of Social Security instead of 85%, and skipped the senior and QBI deductions and the itemized SALT and charity amounts. Its submitted $11,316 does not match its own $10,564 computation."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed Social Security was untaxed and that deductions would bring taxable income near zero. In fact provisional income of about $102,000 makes 85% of benefits ($17,625) taxable, so AGI is $109,149. Even after $34,597 of total deductions, taxable income is $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Filed as a qualifying surviving spouse with joint brackets and a $33,950 standard deduction, but with no dependent child the head files single and itemizes $29,736, which includes about $6,059 of CA income tax in SALT. It also left out the OBBBA senior deduction, phased to $3,951, entirely."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Used qualifying-surviving-spouse joint brackets although the head has no dependent and must file single. It also imputed $12,771 of mortgage interest from the balance even though no interest paid was listed, so that deduction is $0. It took the unphased $6,000 senior deduction instead of $3,951, and its submitted $6,919 does not match its own ~$7,350."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"Limited SALT to property tax and left out the roughly $6,059 of CA income tax, giving $24,222 of itemized deductions instead of $29,736. It also took the full $6,000 senior deduction instead of $3,951 phased by 6% of AGI over $75,000. Its submitted $8,547 is far below single-bracket tax on its own $78,017 of taxable income."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Applied qualifying-surviving-spouse joint brackets and the $33,850 joint standard deduction, plus a non-itemizer charitable deduction. With no dependent child the head files single, itemizes $29,736, and pays 22% on taxable income above $50,400."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA expired for 2026. On that basis it restored personal exemptions, miscellaneous deductions above the 2% floor, and the 15%/25% brackets, all of which OBBBA permanently repealed or replaced. It also imputed $13,835 of mortgage interest that was never given and used surviving-spouse status, then submitted $13,013, which matches none of its own computations."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Used the joint/QSS $32,600 standard deduction although the head has no dependent and files single, where itemizing $29,736 (including about $6,059 of CA income tax in SALT) beats the $18,150 standard deduction. It skipped the $3,951 senior and $910 QBI deductions, and its submitted $6,800 contradicts its own ~$8,700 estimate."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Filed with surviving-spouse joint brackets and the $33,850 standard deduction instead of single, took the full $6,000 senior deduction instead of the $3,951 phased amount, and left out the $910 QBI deduction on rental income."
+us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Applied surviving-spouse joint brackets (2025's $23,850 threshold) and a $32,500 standard deduction. With no dependent the head files single, itemizes $29,736, and also gets the $3,951 phased senior deduction and $910 QBI deduction, none of which it applied."
+us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"Applied pre-TCJA law, with a $5,300 personal exemption, $6,168 of employee business expenses above the 2% floor, and 15%/25% single brackets. OBBBA made the TCJA rates and repeals permanent, so none of these apply. It also understated CA income tax in SALT at $2,019 (about $6,059 is correct) and omitted the $3,951 senior and $910 QBI deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Used the pre-TCJA framework: a $5,200 personal exemption, miscellaneous itemized deductions above the 2% floor, and the old rate schedule. All of these were permanently replaced under OBBBA. It also omitted the senior and QBI deductions and submitted $13,047.69, which contradicts its own stated $12,372."
+us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized only $24,222, leaving the roughly $6,059 of CA income tax out of SALT, and omitted both the $3,951 phased senior deduction and the $910 QBI deduction. That produced taxable income of $84,927 instead of $74,553, and it also used incorrect single bracket thresholds of $12,200 and $49,600."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Applied pre-TCJA law, with a $5,000 personal exemption, $6,168 of unreimbursed employee expenses above the 2% floor, and 10%/15%/25% brackets, all permanently superseded under OBBBA. It left the roughly $6,059 of CA income tax out of SALT and omitted the $3,951 senior deduction."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its stated itemized deductions are mortgage interest and charity. Mortgage interest is $0 because only a loan balance is listed. It omitted the deductions that actually apply: SALT of $14,888 (property tax plus CA income tax), the $3,951 phased senior deduction, and the $910 QBI deduction. Correct taxable income is $74,553 at single rates."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Used pre-TCJA itemized deductions of $33,297 and personal exemptions, which OBBBA permanently eliminated, instead of current law. Under current law, itemized deductions are $29,736, the senior deduction is $3,951 and QBI is $910, all taxed at the 10/12/22% single brackets."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA expired, so it deducted $6,168 of unreimbursed employee expenses above the 2% floor and a $5,050 personal exemption and used pre-TCJA rates. OBBBA made the TCJA structure permanent. It also understated CA income tax at $2,172 and omitted the $3,951 senior and $910 QBI deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no computation. At single 2026 rates, $13,446 corresponds to about $85,000 of taxable income, which is what itemizing only property tax and charity while skipping the senior and QBI deductions produces. The correct deductions total $34,597 (itemized $29,736 including CA income tax, senior $3,951, QBI $910), leaving $74,553 taxable."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at the old $10,000 limit, but the 2026 OBBBA cap is $40,400, so the full $14,888 of property tax plus CA income tax is deductible. It also omitted the $3,951 phased senior deduction, leaving taxable income at $82,846 instead of $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"$9,260 is below the single-bracket tax on any taxable income consistent with its stated deductions of charity and property tax (itemizing $24,222 alone yields over $12,000 of single tax), so it applied lower joint-rate brackets or understated taxable income. The head files single, and single 2026 brackets on $74,553 give $11,113.57."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized the unreimbursed employee business expenses, which OBBBA permanently eliminated as a miscellaneous itemized deduction, and so applied pre-TCJA treatment. Its result omits the roughly $6,059 of CA income tax in SALT, the $3,951 senior deduction and the $910 QBI deduction that bring taxable income to $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"Taxed the head as a qualifying surviving spouse at joint rates although there is no dependent child, so single rates apply. It also subtracted a personal exemption, which has been permanently zero since TCJA/OBBBA, and gave no senior or QBI deduction."
+us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Used single status but left the roughly $6,059 of CA income tax out of SALT, itemizing $24,222 instead of $29,736. It also omitted the OBBBA senior deduction phased to $3,951 and applied 2025 instead of 2026 single brackets, giving taxable income of $84,017 instead of $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Subtracted the $8,351 of unreimbursed employee expenses as an above-the-line adjustment (not allowed), added the $371 tax-exempt pension to AGI, and mis-subtracted to get $84,169 of taxable income. It then subtracted $5,850 of fictitious federal withholding, which is not a nonrefundable credit, to reach $3,240."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"Claimed deductions and offsets reduce tax to zero. AGI is $109,149, and total deductions of $34,597 (itemized $29,736, senior $3,951, QBI $910) leave $74,553 of taxable income. No nonrefundable credit applies, so tax is $11,113.57."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no specific figures. $1,237 corresponds to taxable income of only about $12,000, so it massively overstated deductions or understated the 85% taxable Social Security. The correct single taxable income is $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Used surviving-spouse joint brackets although the head has no dependent and files single. It subtracted a personal exemption that OBBBA permanently set to zero, and its $30,390 itemized total includes the non-deductible employee business expenses instead of the CA income tax in SALT. It also omitted the $3,951 senior and $910 QBI deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Named the right deduction categories but its $10,247 is $866 short at the 22% single rate, which implies about $3,900 more in deductions than allowed. The correct amounts are itemized $29,736 (SALT $14,888 plus charity net of the 0.5% AGI floor), senior deduction phased to $3,951 and QBI $910, for taxable income of $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Applied joint rates for a qualifying surviving spouse and the age-based joint standard deduction. With no dependent child the head files single, itemizes $29,736, and pays 22% on taxable income above $50,400."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Used the surviving-spouse (joint) tax schedule, though without a dependent child the head files single. It also limited itemized deductions to charity and property tax, leaving out the CA income tax in SALT, the $3,951 senior deduction and the $910 QBI deduction."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Treated 'surviving spouse' as qualifying-surviving-spouse status, applying the $33,850 joint standard deduction and joint brackets. That status requires a dependent child, so the head files single, itemizes $29,736, and reaches the 22% bracket above $50,400."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $9,651 is about $1,460 below the correct tax, which implies roughly $6,600 more in deductions than allowed at single rates. Correct itemized deductions are $29,736 (SALT $14,888 plus charity reduced by the 0.5% AGI floor), with the senior deduction at $3,951 and QBI at $910."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Used single status and the phased $3,951 senior deduction, but its $26,456 itemized total understates CA income tax in SALT by about $3,300. It also omitted the $910 QBI deduction on rental income, so taxable income came to $78,742 instead of $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Applied surviving-spouse joint brackets and the $33,850 standard deduction, but with no dependent child the head files single. As a single filer they itemize $29,736 and pay 22% on taxable income above $50,400."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Used surviving-spouse rates and the joint standard deduction instead of single status. It took the unphased $6,000 senior deduction instead of $3,951 and left out the $910 QBI deduction."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Applied a $30,000 joint/QSS standard deduction and joint brackets even though the head has no dependent and files single. It ignored the itemized deductions of $29,736, the $3,951 senior deduction and the $910 QBI deduction, and skipped the age-65 add-on."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA sunset, so it deducted miscellaneous employee expenses above the 2% floor and a $5,419 personal exemption and used old single brackets. OBBBA permanently repealed both deductions and kept the 10/12/22% brackets. It also omitted the $3,951 senior and $910 QBI deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Used the correct single status, phased senior deduction and QBI, but understated CA income tax in SALT at $1,974 (about $6,059 is correct) and used slightly off bracket thresholds. Its taxable income was $78,092 instead of $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Understated CA income tax in SALT at $1,867 instead of about $6,059 and set the QBI deduction to $0 instead of 20% of $4,550 rental income ($910). It also used wrong single bracket thresholds of $12,225 and $49,675 instead of $12,400 and $50,400, giving $79,109 of taxable income instead of $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Counted 100% of Social Security ($20,735) instead of the 85% cap ($17,625). It also applied pre-TCJA law, with miscellaneous employee expenses above the 2% floor and 15%/25% brackets that OBBBA permanently replaced, and omitted the senior and QBI deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"Subtracted an invented ~$901 of rental depreciation from AGI and understated CA income tax in SALT at about $2,024 instead of about $6,059. It also omitted the $910 QBI deduction, ending with $78,538 of taxable income instead of $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value for federal_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Understated CA income tax in SALT at $2,700 instead of about $6,059 and deliberately took no QBI deduction, though 20% of the $4,550 rental income ($910) is deductible. It also skipped the 0.5% AGI floor on charity, leaving taxable income at $78,276 instead of $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"Claimed taxable income was too low for any tax. With $109,149 of AGI (85% of Social Security taxable) and $34,597 of total deductions, taxable income is $74,553 and single tax is $11,113.57."
+us,scenario_022,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized about $26,900 but omitted the OBBBA senior deduction, phased to $3,951, and the $910 QBI deduction, and understated CA income tax in SALT. That left taxable income at about $82,000 instead of $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized only property tax and charity ($24,222), leaving out about $6,059 of CA income tax in SALT, and omitted the $910 QBI deduction on rental income. That produced $80,976 of taxable income instead of $74,553."
+us,scenario_022,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Filed as a surviving spouse at joint rates with the $33,850 joint standard deduction, but the head has no dependent and files single. It also took the full $6,000 senior deduction without the 6% phaseout above $75,000 MAGI and wrongly denied QBI treatment of the rental income."
+us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"Added the taxable $17,625 of Social Security on top of an income total that already included it, inflating AGI to $126,774 instead of $109,149. It also imputed $12,771 of mortgage interest from the loan balance when none was listed and used surviving-spouse joint brackets instead of single."
+us,scenario_022,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"Claimed itemized deductions push taxable income negative by counting imputed mortgage interest, unreimbursed employee expenses and medical costs below the 7.5% AGI floor, none of which are deductible here, plus an invented 'age amount' credit. Allowable itemized deductions are only $29,736, and taxable income is $74,553."
us,scenario_022,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_022,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated Medicare enrollment plus age 77 as sufficient for Medicaid eligibility. Medicare status does not establish Medicaid eligibility, and this person receives no SSI and has medicaid_category NONE, so no California Medicaid pathway applies."
us,scenario_022,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -1720,102 +1739,104 @@ us,scenario_022,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"The model
us,scenario_022,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model used an SDI rate of 1.1% and an outdated wage cap. For 2026, SDI is 1.3% of all wages with no cap, which yields $684.26 rather than $578.99."
us,scenario_022,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"The model stated that no mandatory employee state payroll taxes apply in California. That is wrong: the 1.3% SDI contribution ($684.26) is mandatory and should have been added to the $4,026.58 of federal FICA."
us,scenario_022,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"It used joint/surviving-spouse brackets and joint exemption credits, but the head has no dependent child and must use the single schedule. It also based the 2% employee-expense floor on CA AGI, getting $6,521 instead of about $6,168 (floor based on federal AGI). Its own math reached about $980, yet it reported $3,162 with no derivation."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It left out the $6,168 deduction for unreimbursed employee expenses (allowed in CA after the 2% floor), so taxable income came out at $67,302 instead of $61,134. It then used joint brackets and two personal exemption credits, although a surviving spouse with no dependent files single: $2,274.81 tax less $306 in credits."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It started from an invented $134,529 income, did not exclude Social Security, and took a standard deduction instead of the $30,390 of CA itemized deductions. It also subtracted a made-up $5,320 property-tax credit; the Senior Property Tax Postponement program is not an income tax credit."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It assumed CA taxable income was near zero. In fact, $91,524 of non-Social-Security income less $30,390 of itemized deductions leaves $61,134 taxable, which produces $2,274.81 of single-schedule tax before the $306 in exemption credits."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It used MFJ/surviving-spouse brackets and joint exemption credits, but without a dependent child the head files single. It also left out the deduction for unreimbursed employee expenses above 2% of federal AGI (about $6,168), so taxable income was too high and the rate schedule too low."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It used MFJ/surviving-spouse brackets instead of single brackets and added about $7,000 of mortgage interest that was never given. It also skipped the employee-expense miscellaneous deduction. Its own math gave $1,228 before credits, yet it reported $1,856, which doesn't follow from that figure."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used single status but left out the $6,168 miscellaneous deduction for unreimbursed employee expenses (8,351 less 2% of federal AGI), so taxable income was $67,302 instead of $61,134. Its $2,393 tax figure also fails to match the single schedule for that income."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It used joint brackets and two personal exemption credits plus a senior credit, but a surviving spouse with no dependent files single and gets one personal and one senior credit ($306). It also left out the roughly $6,168 employee-expense deduction, so taxable income was $67,302 instead of $61,134."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It said California follows the TCJA in disallowing unreimbursed employee business expenses. California did not adopt that change, and about $6,168 above the 2% floor is deductible. It also used MFJ/surviving-spouse brackets and a $444 joint exemption credit instead of the single schedule and $306 in single-plus-senior credits."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It used qualifying-widow(er) brackets, but with no dependent the head files single. It also skipped the employee-expense miscellaneous deduction, getting taxable income of $67,302 instead of $61,134. It subtracted no exemption credits, saying they were 'embedded'."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It used joint/QSS brackets instead of single, because it didn't recognize that surviving-spouse status requires a dependent child. It also left out the roughly $6,168 deduction for unreimbursed employee expenses above 2% of federal AGI, so taxable income came out at $67,300."
-us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It started from federal AGI of $109,149 and never subtracted the $17,625 of taxable Social Security, which California excludes. It also left out the roughly $6,168 employee-expense deduction, so taxable income was $84,927 instead of $61,134."
-us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It got taxable income right at $61,134, but its bracket math gave $2,186 instead of the correct single-schedule tax of $2,274.81. It also subtracted only a $167 personal credit and left out the $153 senior exemption credit that applies at age 77."
-us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,age_disability,False,"Its explanation subtracts only one $153 personal exemption credit from $2,288 to get $2,135, leaving out the $153 senior credit for a 77-year-old. The value it submitted, $1,964.52, contradicts that explanation and is $4.29 below the correct $2,274.81 tax minus $306 in credits."
-us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It left out the roughly $6,168 deduction for unreimbursed employee expenses (allowed in CA above the 2% floor), so it taxed $67,302 instead of $61,134. Its bracket breakpoints also differ from the CA single schedule (1% to $11,079, 2% to $26,264, 4% to $41,452, 6% to $57,542, then 8%)."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It based the 2% miscellaneous-deduction floor on CA AGI ($91,524), getting $6,520.52, but the floor uses federal AGI ($109,149), which leaves about $6,168. So it understated taxable income by about $353 ($60,781 vs $61,134). It also used a $310 credit instead of $306 and slightly different bracket figures."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It took the single 65+ standard deduction instead of itemizing $30,390 (property tax, charity, and employee expenses above the 2% floor), which is far larger than the $5,706 CA standard deduction. It gave no supporting bracket calculation for its $1,175."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It based the 2% employee-expense floor on CA AGI, getting $30,743 of itemized deductions instead of $30,390 (floor on federal AGI). Its $2,180 is also higher than single-schedule tax on its own $60,781 less the full $306 in personal-plus-senior credits, which means it did not fully apply both exemption credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It based the 2% employee-expense floor on CA AGI instead of federal AGI, getting taxable income of $60,781 instead of $61,134. It then overstated tax at $2,415, which is above the single schedule's roughly $2,247 on that income, and used $288 in credits instead of $306."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path is CA AGI of $91,524 less $30,390 itemized = $61,134 taxable, which is $2,274.81 on the single schedule minus $306 in credits. Its $3,211 matches a computation that skipped most of the itemized deductions or exemption credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It based the 2% floor on CA AGI, getting $30,743 of itemized deductions instead of the $30,390 that a federal-AGI floor gives. Its $2,035 is also higher than the single-schedule tax less the full $306 in personal-plus-senior credits, so the senior exemption credit was not fully applied."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $2,652 is about $683 too high. The correct result is $2,274.81 of single-schedule tax on $61,134 less $306 in credits. Leaving out the roughly $6,168 deduction for unreimbursed employee expenses above 2% of federal AGI and taxing about $67,302 on the single schedule gives about $2,462 after the $306 in credits, which is still below its $2,652."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,age_disability,False,"Its $2,050 is $81 above the correct result: single-schedule tax of $2,274.81 on $61,134 less the $153 personal and $153 senior exemption credits. That matches netting less than the full $306 in credits, with the 2% employee-expense floor based on CA AGI."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,household_unit_or_filing_status,False,"It based the 2% floor on CA AGI ($30,742.52 itemized instead of $30,390). Its $1,285.40 is far below the single-schedule result, which shows it used joint-type brackets or joint-sized exemption credits; with no dependent child, the head files single."
-us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It invented $24,556 of rental income and $11,326 of mortgage interest, and included Social Security in the CA base. It also used a standard deduction instead of the $30,390 of CA itemized deductions, which made its result wildly too high."
-us,scenario_022,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used a standard deduction instead of the $30,390 of CA itemized deductions, which are much larger, and added tax-exempt pension income back to CA AGI. It taxed $95,830 instead of $61,134, and it applied no personal or senior exemption credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed age-related deductions wipe out the tax, but California has no senior income exclusion beyond the $153 senior credit. $61,134 of taxable income produces $2,274.81 of tax, which is reduced only to $1,968.81 by the $306 in exemption credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"It gave no derivation. Its $1,036 is close to the joint/surviving-spouse bracket results other models produced, while the correct method uses the single schedule (no dependent child), giving $2,274.81 on $61,134 less $306 in credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It used joint/surviving-spouse brackets ($1,136 tax) and about $486 of joint-sized exemption credits. Without a dependent child, the head files single: $2,274.81 tax less $306 in credits. It also based the 2% employee-expense floor on CA AGI instead of federal AGI."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $2,893 is far above the correct $1,968.81. Leaving out the roughly $6,168 employee-expense miscellaneous deduction and subtracting only a $153 personal exemption credit would give about $2,615 on the single schedule, still below its $2,893."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It explicitly used qualifying-surviving-spouse rates. That status requires a dependent child, so the head files single, and the joint-width brackets cut tax to about one-third of the correct single-schedule figure."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse (joint) tax rates. With no dependent the head files single, so tax on $61,134 is $2,274.81, and that should be reduced by both the $153 personal and $153 senior credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It used the surviving-spouse schedule and a $306 joint personal credit plus a $153 senior credit. A single filer gets $153 + $153 and pays tax on the single schedule. It also based the 2% employee-expense floor on CA AGI, getting $30,742.52 instead of $30,390."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,age_disability,False,"It subtracted only the personal exemption credit and left out the $153 senior exemption credit for a 77-year-old. That leaves its result about $190 above the correct $2,274.81 − $306."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,age_disability,False,"It based the 2% employee-expense floor on CA AGI, getting about $60,800 taxable instead of $61,134. It also subtracted only the personal exemption credit and left out the $153 senior exemption credit."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse brackets and $474 of joint-sized personal and senior credits. With no dependent child the head files single, so the correct figure is $2,274.81 of single-schedule tax less $306 in credits. It also based the 2% floor on CA AGI."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse rates. That status requires a dependent child, so the head files single, and the single schedule gives $2,274.81 on $61,134 before $306 in credits."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It declined to compute and defaulted to zero. The required computation is CA AGI of $91,524 less $30,390 itemized = $61,134, taxed at $2,274.81 on the single schedule less $306 in exemption credits."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,llm_error,age_disability,False,"It subtracted only the $153 personal exemption credit and left out the $153 senior exemption credit that applies at age 77. It also based the 2% employee-expense floor on CA AGI, which understated taxable income by about $353."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its taxable income ($61,134) and credits ($306) are correct, but it estimated tax before credits as 'about 2,280' instead of computing $2,274.81 exactly from the single brackets (1% to $11,079, 2% to $26,264, 4% to $41,452, 6% to $57,542, 8% above). That left it $5 too high."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It based the 2% employee-expense floor on CA AGI, getting $6,521 instead of about $6,168 with a floor on federal AGI, so taxable income came out at $60,781 instead of $61,134. Its projected brackets then understated tax at $2,173 instead of $2,274.81."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"It said no nonrefundable credits apply and skipped the $153 personal and $153 senior exemption credits, which apply to this household. It also based the 2% employee-expense floor on CA AGI instead of federal AGI."
-us,scenario_022,state_income_tax_before_refundable_credits,inkling,llm_error,age_disability,False,"It got CA AGI wrong at $90,623; the correct figure is $91,524 because only the $17,625 of taxable Social Security is subtracted. It also subtracted only a $154 personal credit and left out the $153 senior exemption credit."
-us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or explanation for state_income_tax_before_refundable_credits, so nothing could be parsed."
-us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It left out the deduction for unreimbursed employee business expenses above 2% of federal AGI (about $6,168), which California still allows, so it taxed $67,302 instead of $61,134. It also subtracted only a $144 personal credit and missed the $153 senior exemption credit."
-us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed income falls below the tax threshold after deductions. In fact, $91,524 of CA AGI less $30,390 of itemized deductions leaves $61,134 taxable, producing $2,274.81 of single-schedule tax before $306 in credits."
-us,scenario_022,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It counted only property tax and charity ($24,222) and left out the roughly $6,168 deduction for unreimbursed employee expenses above 2% of federal AGI. That made taxable income $67,300 instead of $61,134."
-us,scenario_022,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It left out the roughly $6,168 employee-expense miscellaneous deduction (allowed in CA after the 2% floor on federal AGI), so it taxed $67,302 instead of $61,134. It also used $267 in exemption credits instead of $306."
-us,scenario_022,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It stated that California taxes federally taxable Social Security and kept the $17,625 in CA AGI. California fully excludes Social Security, so AGI is $91,524. It also used MFJ brackets and left out the employee-expense deduction."
-us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It used a roughly $10,726 standard deduction instead of the $30,390 of CA itemized deductions, applied made-up brackets starting at 4%, and subtracted an inflated $639 of credits. The correct credits are $153 personal plus $153 senior."
-us,scenario_022,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed deductions and senior credits wipe out the tax. Itemized deductions of $30,390 still leave $61,134 taxable and $2,274.81 of single-schedule tax, which the $306 in exemption credits reduce only to $1,968.81."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"It treated the head as a qualifying surviving spouse and used joint brackets and joint exemption credits, even though there is no dependent child, so filing status is single. Its own arithmetic came to about $980, then it jumped to $3,162 with no supporting calculation."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It used joint brackets and two personal exemption credits. With no dependent, the surviving spouse files single. It also left out the $6,168 of unreimbursed employee expenses above the 2% floor, which California still allows, so taxable income came to $67,302 instead of $61,134."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,household_unit_or_filing_status,False,"It applied joint brackets and two personal exemption credits instead of the single schedule, which governs because there is no dependent. It also applied the federal 0.5%-of-AGI floor to charity, which California does not have, and left out the $6,168 miscellaneous employee-expense deduction."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It kept Social Security in income instead of using California AGI of $91,524. It used a federal standard deduction instead of California itemized deductions of $30,390. It then subtracted an invented $5,320 property-tax credit from the Senior Property Tax Postponement program, which is not an income tax credit."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed taxable income was near zero. In fact $91,524 of California AGI minus $30,390 of itemized deductions leaves $61,134 taxable. Single-bracket tax on that is $2,274.81, and $306 of exemption credits cannot eliminate it."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It used MFJ/surviving-spouse brackets and joint exemption credits, but a surviving spouse with no dependent files single. It also never deducted the $6,168 of unreimbursed employee expenses above the 2% floor, so it understated tax by using the wide low-rate joint brackets."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It applied MFJ brackets for a surviving spouse with no dependent, who should be taxed on the single schedule. It also added an unlisted $7,000 of mortgage interest. Its own $1,228 bracket tax then rose to $1,856 after subtracting credits, which contradicts its arithmetic."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used the correct single schedule and both exemption credits, but left out the $6,168 of unreimbursed employee business expenses above the 2% floor. California still allows that miscellaneous itemized deduction, so its taxable income was $67,302 instead of $61,134."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It used joint brackets and two personal exemption credits plus a senior credit, about $471, although with no dependent the head files single and gets $306. It also left out the $6,168 miscellaneous employee-expense deduction."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It treated the surviving spouse as MFJ and used joint brackets, but with no dependent the head files single. It also claimed wrongly that California conforms to TCJA on employee expenses, so it dropped the $6,168 miscellaneous deduction, and it subtracted an inflated $588 of exemption credits."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It left out the $6,168 of unreimbursed employee expenses above the 2% floor, which California allows, and so taxed $67,302. It guessed the tax from an average-rate assumption on qualifying-widow brackets instead of the single schedule, and claimed the senior exemption credit was 'embedded' in the tables instead of subtracting $306 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It applied joint/QSS brackets although the head has no dependent and files single. It also left out the $6,168 miscellaneous employee-expense deduction, taxing $67,300 instead of $61,134."
+us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It started from federal AGI of $109,149 and did not subtract the $17,625 of taxable Social Security that California excludes. It also left out the $6,168 miscellaneous employee-expense deduction, so taxable income came to $84,927 instead of $61,134."
+us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,age_disability,False,"It reached the correct taxable income of $61,134, but subtracted only a single $167 personal exemption credit and omitted the $153 senior exemption credit for the 77-year-old head. It also understated single-bracket tax as $2,186 instead of $2,274.81."
+us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It had the correct $61,134 of taxable income. Its submitted $1,964.52 implies $2,270.52 of pre-credit tax instead of $2,274.81 from the 2026 single bracket thresholds. Its explanation also contradicts the value, citing $2,288 less a single $153 credit, which is $2,135."
+us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It itemized only charity and property tax ($24,222) and left out the $6,168 of unreimbursed employee expenses above the 2% floor that California allows. That gave taxable income of $67,302 instead of $61,134, and its bracket arithmetic also overshot the single schedule."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It applied the 2% floor for miscellaneous deductions to California AGI ($91,524) instead of federal AGI ($109,149), overstating the employee-expense deduction by about $353. It also subtracted $310 of exemption credits instead of $306."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It used a standard deduction for a single filer age 65 or over instead of itemizing $30,390 (property tax, charity, and miscellaneous employee expenses), which far exceeds the $5,706 standard deduction. Its result does not follow from the single-bracket tax of $2,274.81 on $61,134."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its taxable income was about $60,781 (it applied the 2% floor to California AGI), on which single-bracket tax is about $2,247, or about $1,941 after $306 of credits. Its $2,180 overstates the bracket tax from mis-indexed 2026 single thresholds."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It stated $2,415 of tax before credits on $60,781, but the 2026 single schedule gives about $2,247 there ($2,274.81 on the correct $61,134). It also subtracted $288 instead of $306 of personal and senior exemption credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,211 implies about $3,517 of pre-credit tax, or roughly $15,000 more taxable income than the correct $61,134. That means it did not take the full $30,390 of California itemized deductions from $91,524 of AGI."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used $30,743 of itemized deductions (2% floor on California AGI), giving about $60,781 taxable. Single-bracket tax there is about $2,247 less $306 of credits, about $1,941, so its $2,035 reflects wrong 2026 single bracket thresholds."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $2,652 implies taxable income around $67,000–$70,000, which matches itemizing only property tax and charity and leaving out the $6,168 of unreimbursed employee expenses above the 2% floor that California allows."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its stated method matches the reference: exclude Social Security, itemize, and subtract exemption credits. But its $2,050 implies about $2,356 of pre-credit tax instead of the $2,274.81 that the 2026 single brackets give on $61,134, which points to mis-indexed bracket thresholds."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"Its taxable income was nearly correct at $60,781, but its $1,285.40 is far below the single-schedule result of about $2,247 less $306. It understated the bracket tax by not applying the single 6% and 8% brackets that start near $42,000 and $58,000."
+us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It invented $24,556 of rental income and $11,326 of mortgage interest, did not exclude Social Security from California AGI, and used a standard deduction instead of $30,390 of itemized deductions. Its $11,626 has no connection to the $61,134 taxable-income base."
+us,scenario_022,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It added a fabricated $1,750 pension adjustment to reach $102,919 of California AGI, and used a $7,089 standard deduction instead of the $30,390 itemized total. It also skipped the $306 of personal and senior exemption credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that age and deductions fully offset the tax. In fact $91,524 of California AGI minus $30,390 of itemized deductions leaves $61,134 taxed at $2,274.81, of which the $306 exemption credits remove only a small part."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave no derivation. On $61,134 of taxable income, the single schedule gives $2,274.81 less $306 of credits. Its $1,036 is consistent with using the low joint brackets or understating the 6%/8% bracket tax."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It applied joint/surviving-spouse brackets ($1,136 of tax) and $486 of joint-style exemption credits. With no dependent, the head files single, so tax is about $2,247 on its taxable income, less $306 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"It subtracted only the personal exemption credit and dropped the $153 senior exemption credit for the 77-year-old head. Its $2,893 is also about $900 high, which reflects understated itemized deductions relative to the $30,390 total."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It applied qualifying-surviving-spouse rates, but that status requires a dependent child and none is listed, so the single schedule and $306 of credits apply. The joint brackets cut its tax to $657."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It taxed the head on the surviving-spouse (joint) schedule, but the head has no dependent and must use the single brackets. It also mentioned only the personal exemption credit and not the senior credit."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It used the surviving-spouse schedule, giving $1,159 of tax, and a doubled $306 personal credit plus $153 senior credit. With no dependent the head files single: about $2,247 of bracket tax less a single $153 personal credit and a $153 senior credit."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,age_disability,False,It subtracted only the personal exemption credit and left out the $153 senior exemption credit owed to the 77-year-old head. That left its result about $190 above the reference.
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,age_disability,False,"It reached about $60,800 of taxable income, but subtracted only the personal exemption credit. Its $2,095 equals about $2,247 of single-bracket tax less $153, so it omitted the second $153 senior exemption credit."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse (joint) brackets and $474 of credits, but with no dependent child the head files single. That means single brackets and $306 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,household_unit_or_filing_status,False,"It applied surviving-spouse rates to a head with no dependent, who files single. The joint brackets push its tax down to $650."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It declined to compute and defaulted to $0. The correct path is $91,524 of California AGI minus $30,390 of itemized deductions, $2,274.81 of single-bracket tax, and $306 of exemption credits."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,llm_error,age_disability,False,It subtracted only the $153 personal exemption credit and left out the $153 senior exemption credit for the 77-year-old head. It also applied the 2% miscellaneous floor to California AGI instead of federal AGI.
+us,scenario_022,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It reproduced the $61,134 of taxable income and the $306 of credits exactly, but rounded the single-bracket tax to 'about 2,280' instead of computing $2,274.81 from the 2026 thresholds, leaving it $5 high."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied the 2% miscellaneous floor to California AGI instead of federal AGI, overstating deductions by $353. It also used projected single brackets that gave $2,173 instead of about $2,247 of tax on $60,781."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"It stated that no nonrefundable credits apply and never subtracted the $153 personal and $153 senior exemption credits ($306), which reduce single-bracket tax to $1,968.81."
+us,scenario_022,state_income_tax_before_refundable_credits,inkling,llm_error,age_disability,False,"It subtracted only a $154 personal exemption credit and left out the $153 senior exemption credit for the 77-year-old head. It also understated California AGI as $90,623 instead of $91,524."
+us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score."
+us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It itemized only property tax and charity ($24,222) and left out the $6,168 of unreimbursed employee expenses above the 2% floor that California allows. It also subtracted only one $144 personal credit and missed the senior exemption credit."
+us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed income falls below the threshold after the standard deduction. In fact California AGI of $91,524 minus $30,390 of itemized deductions leaves $61,134 of taxable income, and the tax on that is well above zero."
+us,scenario_022,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It itemized only property tax and charity and left out the $6,168 of employee expenses above the 2% floor, so it taxed $67,300 instead of $61,134. It also cited the 9.3% bracket, which single filers do not reach at that income."
+us,scenario_022,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It left out the $6,168 miscellaneous deduction for unreimbursed employee expenses that California allows, which kept taxable income at $67,302. It also subtracted $267 instead of $306 of exemption credits."
+us,scenario_022,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It wrongly claimed California taxes federally taxable Social Security and used $109,149 of federal AGI. It also left out the $6,168 miscellaneous deduction and applied MFJ brackets to a head with no dependent who files single."
+us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It took a $10,726 standard deduction instead of the $30,390 of California itemized deductions. It used invented brackets that start at 4%, and its subtraction of $639 of credits does not reconcile with its own bracket sum."
+us,scenario_022,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It asserted that deductions and the senior exemption credit eliminate the tax. In fact $61,134 of taxable income produces $2,274.81 of single-bracket tax, and the $306 of exemption credits leave $1,968.81."
us,scenario_022,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly found about $644 of tax on $6,435 of taxable income. It then removed that tax with a $1,000 Saver's Credit and an AOTC. Form 8880 subtracts the $8,000 403(b) distribution from contributions, which makes the Saver's Credit $0. The AOTC needs qualified tuition expenses, and none are listed, so it is also $0. The correct result is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"The model correctly computed $643.50 of tentative tax and correctly set the AOTC to 0. It then applied a $1,000 Saver's Credit without subtracting the $8,000 taxable 403(b) distribution from eligible contributions as Form 8880 requires. That distribution exceeds every contribution, so the credit is $0 and the tax stays at $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model left the $8,000 taxable 403(b) distribution out of gross income. It also used the outdated $14,600 standard deduction instead of the 2026 amount of $16,100. With the distribution included, AGI is $22,534 and taxable income is $6,434.34, which gives $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model's reasoning counts only wages minus the retirement deferrals against the standard deduction. It omits the $8,000 taxable 403(b) distribution. With the distribution, AGI is $22,534 and taxable income after the $16,100 standard deduction is $6,434.34. That produces $643.43 of tax, and no nonrefundable credit offsets it."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly reached about $644 of tax. It then applied a $1,500 nonrefundable AOTC even though no qualified tuition or fee expenses are listed. The AOTC is a percentage of qualified expenses, so with $0 of expenses the credit is $0 and the tax stays at $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model assumed the head has qualified 1098-T expenses that support the maximum AOTC, but no expense amount is given and an unlisted amount counts as 0. With $0 of expenses, the AOTC is $0. The $644 of tax on $6,434 of taxable income therefore stands at $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model computed the correct AGI and tentative tax. It then applied up to $1,500 of nonrefundable AOTC with no qualified tuition expenses listed. The AOTC is a percentage of qualified expenses, so $0 of expenses means a $0 credit and $643.43 of tax remains."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"The model explicitly assumed the 403(b) distributions do not reduce Saver's Credit contributions. Form 8880 line 5 does subtract distributions received during the testing period, so the $8,000 distribution reduces the $3,599 of contributions to $0. That makes the credit $0 and leaves $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model got AGI and the zero credits right but estimated the 2026 single standard deduction at $15,350 by inflating the 2025 amount. OBBBA set the 2026 amount at $16,100. That deduction gives taxable income of $6,434.34 and tax of $643.43, not $719."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model overstated gross income at $26,443 yet still claimed taxable income is near $0. It then applied the nonrefundable AOTC even though there are no qualified tuition expenses. In fact $6,434.34 of income is taxable after the $16,100 standard deduction, and the AOTC is $0, so the tax is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"The model correctly computed about $644 of tax and correctly set the AOTC to $0. It then applied a 50% Saver's Credit on $2,000 of contributions without netting out the $8,000 403(b) distribution, as Form 8880 requires. After that offset, eligible contributions and the credit are $0, leaving $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 of employer-sponsored insurance premiums from taxable wages as a pre-tax exclusion. That cut AGI to $14,446, below the standard deduction. PolicyEngine subtracts listed pre-tax payroll deductions from wages, and the only one this household lists is the traditional 401(k) deferral, so AGI is $22,534 and $6,434.34 of income is taxed at 10%."
-us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model excluded the $8,089 of ESI premiums from wages, which gave an AGI of $14,446 instead of $22,534. It also used a TCJA-sunset standard deduction of $8,350 instead of OBBBA's $16,100. The two errors partly offset each other and produced $609.60 instead of $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model correctly computed AGI and correctly found both credits to be $0. However, it assumed TCJA expired and used an $8,300 standard deduction plus a $5,300 personal exemption. OBBBA made the higher standard deduction permanent at $16,100 for 2026 and kept the exemption at $0, so taxable income is $6,434.34, not $8,935."
-us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"The model applied a $1,000 Saver's Credit without reducing contributions by the $8,000 403(b) distribution, which makes the credit $0 under Form 8880. It also used the 2025 $15,750 standard deduction instead of $16,100. The correct tax after nonrefundable credits is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model applied a 50% Saver's Credit of $1,000 without subtracting the $8,000 403(b) distribution from contributions, which reduces eligible contributions to $0. It also used a pre-OBBBA standard deduction plus exemption of about $14,000 instead of $16,100. No credit offsets the $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model asserted that taxable income is $0, but AGI of $22,534 (including the $8,000 taxable 403(b) distribution) exceeds the $16,100 standard deduction by $6,434.34. That amount taxed at 10% is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model relied on the Saver's Credit to offset the tax. Form 8880 nets the $8,000 403(b) distribution against the $3,599 of contributions, so eligible contributions and the credit are both $0. The $643.43 of tax on $6,434.34 of taxable income remains."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's AGI of $14,446 shows it subtracted the $8,089 of ESI premiums from wages, but the only pre-tax payroll deduction this household lists is the traditional 401(k) deferral, so AGI is $22,534. The model also applied a Saver's Credit that the $8,000 distribution reduces to $0. The correct tax is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"The model claimed nonrefundable credits eliminate the tax, but none apply. The $8,000 403(b) distribution reduces the Saver's Credit to $0 under Form 8880, and the AOTC is $0 without qualified tuition expenses. Taxable income of $6,434.34 at 10% gives $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model subtracted a $200 Saver's Credit even though the $8,000 403(b) distribution reduces eligible contributions to $0. Its pre-credit tax of $753.50 implies a $15,000 standard deduction instead of OBBBA's $16,100. The correct tax is $643.43 with no credits."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model claimed the deductions bring taxable income to zero. After the traditional 401(k) deferral, the $8,000 taxable 403(b) distribution, the $130 IRA deduction and the $16,100 standard deduction, $6,434.34 remains taxable. That yields $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model correctly computed AGI of $22,535 and correctly found both the AOTC and Saver's Credit to be $0. Its $754 of tax implies a $15,000 standard deduction (the pre-OBBBA 2025 figure) rather than the 2026 amount of $16,100. With $16,100, taxable income is $6,434.34 and the tax is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly netted the $8,000 distribution against contributions to zero out the Saver's Credit and correctly set the AOTC to 0. It then used an estimated $15,400 standard deduction instead of the 2026 OBBBA amount of $16,100. That overstated taxable income by $700 and gave $713.50 instead of $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"The model's reasoning is incoherent. It invents a $500 distribution exclusion, refuses to exclude the 401(k) deferral, uses a $15,000 standard deduction and switches between AOTC amounts. It ends with a negative value of -$997, which is impossible for tax after nonrefundable credits. Correct AGI is $22,534, taxable income is $6,434.34 and the tax is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model relied on an AOTC-related nonrefundable credit to eliminate the tax, but no qualified tuition expenses are listed, so the AOTC is $0. It also claimed the standard deduction offsets the income, but $6,434.34 remains taxable. The result is $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model applied part of a nonrefundable AOTC to reduce the tax to $121, but the AOTC is $0 because no qualified education expenses are listed. With no nonrefundable credits, the 10% tax on $6,434.34 of taxable income stays at $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"The model used the Saver's Credit to offset the tax but ignored the Form 8880 rule that subtracts distributions received during the testing period. The $8,000 403(b) distribution exceeds every contribution, so the credit is $0 and $643.43 of tax remains."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"The model credited the retirement-savings credit against the tax without reducing contributions by the $8,000 taxable 403(b) distribution. That reduction leaves zero eligible contributions, so the credit is $0 and the tax is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model treated the $8,089 of employer-sponsored insurance premiums as pre-tax, which gave an AGI of $14,446. The only pre-tax payroll deduction this household lists is the $2,778 traditional 401(k) deferral, so AGI is $22,534. After the $16,100 standard deduction, $6,434.34 is taxable at 10%, which is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,"The model correctly set the AOTC to zero but offset the tax with a retirement savings credit. Form 8880 subtracts the $8,000 403(b) distribution from contributions, which makes the credit $0. The $643.43 of tax remains."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated wages as below the standard deduction and ignored the $8,000 taxable 403(b) distribution. That distribution raises AGI to $22,534 and taxable income to $6,434.34. No nonrefundable credit applies, so the tax is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly computed AGI of $22,535 and correctly set the credits to $0. It then assumed TCJA expired and used an $8,500 standard deduction plus a $5,300 personal exemption. OBBBA set the 2026 single standard deduction at $16,100 with no personal exemption, so taxable income is $6,434.34 and the tax is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"The model offset about $890 of tax with a $1,000 Saver's Credit without subtracting the $8,000 403(b) distribution, which reduces eligible contributions to $0. It also applied a pre-OBBBA standard deduction plus personal exemption instead of the $16,100 standard deduction. The correct tax is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model correctly zeroed out the Saver's Credit by netting the $8,000 distribution against contributions and correctly set the AOTC to 0. It then assumed TCJA expired, using an $8,500 standard deduction and a $5,450 exemption. Under OBBBA the 2026 deduction is $16,100 with no exemption, which gives $6,434.34 of taxable income and $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed TCJA sunset and used an $8,500 standard deduction with a pre-TCJA 15% bracket, which put taxable income at $14,035. OBBBA made the $16,100 standard deduction and the 10%/12% brackets permanent for 2026. Taxable income is $6,434.34, all at 10%, which is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,089 of employer health premiums from wages, which gave an AGI of $14,446 below the $16,100 standard deduction. The only pre-tax payroll deduction this household lists is the traditional 401(k) deferral, so AGI is $22,534 and $6,434.34 is taxable, giving $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value for federal_income_tax_before_refundable_credits. The correct derivation is AGI of $22,534 minus the $16,100 standard deduction, which leaves $6,434.34 taxed at 10%, or $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model submitted no value for federal_income_tax_before_refundable_credits. The correct derivation is AGI of $22,534 minus the $16,100 standard deduction, which leaves $6,434.34 taxed at 10%, or $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model applied a $1,500 nonrefundable AOTC even though no qualified tuition expenses are listed, so the AOTC is $0. It also used a $15,400 standard deduction instead of $16,100. The correct tax is $643.43 with no credits applied."
-us,scenario_023,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model followed the correct path and found no usable nonrefundable credits, but it reported $6,470 of taxable income. That figure implies a standard deduction of about $16,065. AGI of $22,534.34 minus the $16,100 standard deduction is $6,434.34, so the tax is $643.43, not $647."
-us,scenario_023,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model correctly computed $644 of tax and correctly set the AOTC to 0. It then applied a $1,000 Saver's Credit on $2,908 of contributions without subtracting the $8,000 taxable 403(b) distribution as Form 8880 requires. That subtraction reduces the credit to $0 and leaves $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model applied the nonrefundable AOTC to eliminate about $678 of tax, but no qualified tuition expenses are listed, so the AOTC is $0. It also used a $15,750 standard deduction instead of $16,100. The correct tax is $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model claimed income is too low to tax after the standard deduction and leaned on the AOTC. In fact $6,434.34 remains taxable after the $16,100 deduction, and the AOTC is $0 because no qualified expenses are listed. The tax is $643.43."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"It correctly computed about $644 of pre-credit tax but then zeroed it with a $1,000 Saver's Credit and an AOTC. The $8,000 403(b) distribution offsets all $3,599 of retirement contributions on Form 8880, so the Saver's Credit is $0. No qualified tuition is listed, so the AOTC is also $0."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,categorical_eligibility,False,"It computed the $643.50 pre-credit tax correctly, then applied a $1,000 Saver's Credit without netting the $8,000 taxable 403(b) distribution against contributions. Under §25B(d)(2), that distribution brings eligible contributions to zero, so no credit is available."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,categorical_eligibility,False,"It correctly found $643.50 of tax but treated the Saver's Credit as 50% of $2,000. It never subtracted the $8,000 retirement distribution from eligible contributions, and that subtraction eliminates the credit, leaving $643.43 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left the $8,000 taxable 403(b) distribution out of income and used the outdated $14,600 standard deduction, which produced negative taxable income. The correct AGI is $22,534. After the $16,100 deduction, $6,434 is taxable at 10%."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"Its reasoning counted only wages minus deferrals and the standard deduction, and it left out the $8,000 taxable 403(b) distribution. Including it gives AGI of $22,534, which exceeds the $16,100 standard deduction by $6,434, so the tax is $643.43, not zero."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"It computed about $644 of tax correctly, then offset it with a $1,500 AOTC nonrefundable portion. No qualified tuition or fees are listed (treated as $0), so the AOTC is $0 and the full $643.43 remains."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It assumed qualified 1098-T expenses existed and applied the AOTC nonrefundable portion. Having a 1098-T and half-time enrollment does not create expenses, and with $0 qualified tuition the AOTC is $0, so the $643.43 tax stands."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"It reached the correct $6,435 taxable income and $644 tax, then wiped the tax out with up to $1,500 of AOTC. The credit is computed from qualified education expenses, which are $0 here, so the AOTC is zero."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,categorical_eligibility,False,"It explicitly assumed the 403(b) distributions do not reduce eligible contributions. Form 8880 subtracts testing-period distributions from contributions, and the $8,000 distribution exceeds the $3,599 contributed, so the Saver's Credit is $0 and the tax is $643.43."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It estimated the 2026 single standard deduction by inflating $15,000 to $15,350, but the OBBBA 2026 amount is $16,100. That overstated taxable income at $7,185 instead of $6,434, giving $719 instead of $643.43."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It claimed taxable income was near $0 after the standard deduction, but $22,534 AGI minus $16,100 leaves $6,434. It then applied an AOTC even though qualified education expenses are $0, so the $643.43 tax was wrongly offset."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It computed $644 of tax correctly but applied a $1,000 Saver's Credit at the 50% rate. It ignored the rule that the $8,000 retirement distribution reduces eligible contributions to zero."
+us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,089 employer-sponsored insurance premium from the listed gross wages as a pre-tax exclusion, which pushed AGI down to $14,446. Only the $2,778 traditional 401(k) deferral reduces wages here, so AGI is $22,534, leaving $6,434 taxable after the $16,100 deduction."
+us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,089 ESI premium from wages, which is not an exclusion here, and used an $8,350 TCJA-sunset standard deduction instead of the $16,100 2026 amount. The correct taxable income is $6,434, not $6,096."
+us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed TCJA had expired and used an $8,300 standard deduction plus a $5,300 personal exemption. Under OBBBA, the 2026 single standard deduction is $16,100 with no personal exemption, so taxable income is $6,434, not $8,935."
+us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It used the 2025 $15,750 standard deduction instead of the 2026 $16,100 amount. It then applied a $1,000 Saver's Credit without offsetting the $8,000 403(b) distribution against contributions, and that offset eliminates the credit."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It used about $14,000 of deduction plus personal exemption instead of the $16,100 standard deduction. It then offset the tax with a $1,000 Saver's Credit, ignoring that the $8,000 retirement distribution zeroes out eligible contributions."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It asserted taxable income is zero, but AGI of $22,534 (including the $8,000 taxable 403(b) distribution) exceeds the $16,100 standard deduction. That leaves $6,434 taxed at 10%."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It relied on the Saver's Credit to eliminate the tax. Under §25B(d)(2), the $8,000 403(b) distribution exceeds the $3,599 in contributions, so the credit is $0 and the $643.43 tax remains."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It produced AGI of $14,446 by subtracting the $8,089 employer insurance premium from gross wages, which is not an exclusion here. It then applied a Saver's Credit that the $8,000 distribution offset eliminates. The correct AGI is $22,534, and the tax is $643.43 with no credit."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It attributed a zero result to the standard deduction and nonrefundable credits, but the $16,100 deduction leaves $6,434 taxable. No nonrefundable credit applies: there are no AOTC expenses, and the Saver's Credit is wiped out by the distributions."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $553.50 result implies $753.50 of pre-credit tax, consistent with a $15,000 standard deduction instead of $16,100. It also subtracted a $200 Saver's Credit that the $8,000 retirement distribution eliminates."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It claimed zero taxable income, which ignores that wages after the 401(k) deferral plus the $8,000 taxable 403(b) distribution produce $22,534 AGI. That is $6,434 above the $16,100 standard deduction."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly zeroed both the AOTC and the Saver's Credit, but its $754 implies a $15,000 standard deduction (taxable income $7,535). The 2026 OBBBA single amount is $16,100."
+us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It handled AGI and both credits correctly but estimated the 2026 single standard deduction at $15,400 instead of $16,100. That overstated taxable income by $700 and the tax by $70."
+us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It wrongly refused to exclude the $2,778 traditional 401(k) deferral from wages, applied a nonexistent $500 distribution exclusion, used a $15,000 standard deduction, and submitted a negative tax figure, which is impossible. The correct path is $22,534 AGI minus $16,100, taxed at 10%, with no credits."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It used an AOTC nonrefundable credit to eliminate the tax. With no qualified education expenses listed, the AOTC is $0, so the $643.43 tax on $6,434 of taxable income stands."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It applied a partial AOTC nonrefundable credit based on enrollment alone, but the credit requires qualified expenses, which are $0. Its reasoning also never adds the $8,000 taxable 403(b) distribution. The correct result is 10% of $6,434, or $643.43."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,categorical_eligibility,False,"It offset the tax with a Saver's Credit on the listed contributions without netting the $8,000 taxable 403(b) distribution against them. That netting reduces eligible contributions, and the credit, to zero."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"It claimed the retirement savings credit eliminates the tax, but Form 8880 subtracts the $8,000 testing-period distribution from the $3,599 in contributions. That leaves no eligible contributions and no credit."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It treated the $8,089 employer-plan premium as a pretax wage reduction, which pushed AGI down to $14,446 and under the standard deduction. With only the 401(k) deferral excluded, AGI is $22,534 and $6,434 is taxable."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,categorical_eligibility,False,"It correctly rejected the AOTC but used a retirement savings credit to offset the tax. The $8,000 403(b) distribution reduces eligible contributions to zero, so no Saver's Credit applies."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It treated wages as under the standard-deduction threshold and ignored the $8,000 taxable 403(b) distribution. That distribution brings AGI to $22,534, which is $6,434 above the $16,100 deduction."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed TCJA had expired in 2026 and used an $8,500 standard deduction plus a $5,300 personal exemption. OBBBA made the higher deduction permanent at $16,100 for 2026 with no exemption, so taxable income is $6,434, not about $8,600."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,categorical_eligibility,False,"It used a pre-TCJA standard deduction plus a personal exemption to reach about $8,900 of taxable income. It then applied a $1,000 Saver's Credit that the $8,000 retirement distribution eliminates."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It correctly zeroed the Saver's Credit and the AOTC, but assumed TCJA had sunset and used an $8,500 standard deduction plus a $5,450 exemption. The 2026 law provides a $16,100 standard deduction and no personal exemption, so taxable income is $6,434."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It reverted to a pre-TCJA $8,500 standard deduction and a 15% bracket, giving $14,035 of taxable income. Under OBBBA, the $16,100 deduction leaves $6,434 taxable, all in the 10% bracket."
+us,scenario_023,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It subtracted the $8,089 employer health premium from gross wages, which drove AGI down to $14,446 and below the $16,100 deduction. The correct AGI is $22,534, leaving $6,434 taxable and $643.43 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value for federal_income_tax_before_refundable_credits, so there is no answer to score against the $643.43 result."
+us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It submitted no value for federal_income_tax_before_refundable_credits, so there is no answer to score against the $643.43 result."
+us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It used a $15,400 standard deduction instead of $16,100, then offset the tax with a $1,500 AOTC nonrefundable portion. The AOTC is $0 because no qualified education expenses are listed."
+us,scenario_023,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,other,False,"It correctly found $22,535 of AGI and no usable credits, but mis-subtracted the standard deduction, getting about $6,470 of taxable income. The correct figure, $22,534 minus $16,100, is $6,434, which gives $643.43 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It computed $644 of tax correctly, then applied a $1,000 Saver's Credit on $2,908 of contributions. It never subtracted the $8,000 403(b) distribution that Form 8880 nets against contributions, which reduces the credit to zero."
+us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It used the 2025 $15,750 standard deduction instead of $16,100, then let the AOTC eliminate the tax. With $0 qualified education expenses, the AOTC is $0."
+us,scenario_023,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It treated income as too low to owe tax and invoked the AOTC. AGI of $22,534 exceeds the $16,100 standard deduction by $6,434, and the AOTC is $0 because no qualified tuition is listed."
us,scenario_023,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly phased the EITC out to $0 and even noted that zero listed tuition gives a $0 AOTC. It then reversed itself by assuming 'typical qualified expenses default' of at least $4,000 and submitted a $1,000 refundable AOTC. With no qualified expenses, the AOTC is $0."
us,scenario_023,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated the full $2,500 AOTC as refundable. The refundable share is capped at 40% ($1,000). More importantly, the credit base is $0 because no qualified tuition expenses are listed. It also left the $8,000 403(b) distribution out of AGI."
us,scenario_023,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model made up a refundable AOTC of about $400 plus a small EITC to reach $522. No qualified education expenses are listed, so the AOTC is $0. Its own AGI of $22,535 is above the roughly $19,540 point where the 2026 childless EITC reaches zero, so the EITC is also $0."
@@ -1926,160 +1947,164 @@ us,scenario_023,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"Its $2
us,scenario_023,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_023,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_023,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_023,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Treated the head as a SNAP elderly/disabled member, taking a medical deduction and uncapped shelter. It also submitted $3,936, which contradicts its own $2,328 calculation. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Applied the uncapped shelter and medical deductions under the disability reading. It also added a utility allowance for utilities that aren't listed. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Applied the 130% FPL gross income test and declared the household ineligible. California's BBCE uses a 200% FPL screen, which $2,120/mo passes. It also wrongly treated rent as a reason for ineligibility, when rent actually feeds a shelter deduction that gives $877 rounded net income and $34/mo ($408/year) on the capped reading under c_snap_hold_fy2026. This output is excluded from scoring."
-us,scenario_023,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Concluded the household is over SNAP's limits without doing the calculation. Under California's 200% FPL BBCE screen, gross income passes. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Treated the head as SNAP-disabled (uncapped shelter, plus a $1,180 medical deduction) under the disability reading. It then raised its own $2,529 result to $3,768, which is above the one-person maximum. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Gave elderly/disabled treatment, including a medical deduction and uncapped shelter, under the disability reading. It also miscomputed the earned-income deduction as $1,454 a year, then overrode its own $1,788 result with an unexplained $2,549. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Misstated California's 200% FPL BBCE gross limit as about $2,510 and concluded that $2,120/mo fails it. The FY2026 one-person limit is about $2,608, so the household passes. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Assumed the disability flag lifts the shelter cap and allows a medical deduction. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Treated the head as SNAP-disabled, applying a $73 medical deduction and an uncapped $1,224 shelter deduction to reach $328 net income. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring. It also used the FY2025 $292 maximum instead of $298."
-us,scenario_023,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Applied the 130% FPL gross income test, but California's BBCE uses 200% FPL, which this household passes. It also never applied the $744 shelter deduction, which gives $877 rounded net income and $34/mo ($408/year) on the capped reading under c_snap_hold_fy2026. This output is excluded from scoring."
-us,scenario_023,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Applied the medical deduction and an uncapped excess shelter deduction under the disability reading, giving about $346 net income. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Removed the shelter cap under the disability reading and calculated about $446 net income. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Treated the head as SNAP-disabled and took a $747/mo medical deduction, which is inflated by employer-paid premiums, plus an uncapped $1,561 shelter deduction. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Applied a $98 excess medical deduction and an uncapped $1,239 shelter deduction under the disability reading. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Said the disabled head is exempt from the gross test and allowed the medical deduction and uncapped shelter. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Applied the medical deduction and an uncapped shelter deduction that includes an unlisted $595 utility allowance, as if the household were SNAP elderly/disabled. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring. It also used a $291 maximum instead of $298."
-us,scenario_023,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"$3,492 is $291 × 12, consistent with an assumed $291 monthly maximum. The answer gives no calculation of net income or deductions. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Relied on uncapped excess shelter and medical deductions 'due to disability.' On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Lowered net income to about $300 using uncapped shelter and medical deductions for a 'disabled head.' On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Asserted that income or resources exceed SNAP limits. In fact $2,120/mo passes California's 200% FPL BBCE screen, assets are $130, and the capped calculation under c_snap_hold_fy2026 gives $877 rounded net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Used 'uncapped shelter expenses for a disabled individual' under the disability reading. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Applied the uncapped excess shelter deduction for disabled households and brought net income to $0. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Applied the uncapped shelter deduction and the medical deduction on the basis of disability. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gemini-4-argon,prompt_ambiguity,age_disability,False,"Reduced net income to $0 with an uncapped shelter deduction on the grounds that the head is disabled. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring. It also used the FY2025 $292 maximum."
-us,scenario_023,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Applied the $1,180 medical deduction and an uncapped $14,837 annual shelter deduction for disabled households, under the disability reading. It also used a $292 maximum and a $204 standard deduction. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Left out the excess shelter deduction entirely, which is capped at $744/mo on the capped reading. Under c_snap_hold_fy2026, that reading gives $877 rounded net income and $34/mo ($408/year). This output is excluded from scoring. It compared 30% of annual net income with a monthly maximum allotment, then submitted an unsupported $871 that contradicts its own conclusion of zero."
-us,scenario_023,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Concluded SNAP is zero without computing it. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Returned zero because the inputs didn't show benefit receipt, ignoring the instruction to assume take-up. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Treated the household as SNAP-disabled and used the medical-expense and uncapped excess-shelter deductions to reach $0 net income. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Gave the medical and uncapped shelter deductions under the disability reading. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Correctly cleared the student rule through the 24 hours of work, but then applied the medical deduction and the uncapped disability shelter deduction. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Concluded that wages plus the 403(b) distribution eliminate the benefit. Under California's 200% FPL BBCE, $2,120/mo passes, and after the earned-income, $209 standard and $744 capped shelter deductions, net income of $877 after rounding gives $34/mo ($408/year) on the capped reading under c_snap_hold_fy2026. This output is excluded from scoring."
-us,scenario_023,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Applied a $150 standard medical deduction and an uncapped excess shelter deduction for disability, under the disability reading. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Treated the disability flag as unlocking the medical deduction and the uncapped shelter deduction, which reduced net income to zero. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Used an uncapped shelter deduction 'with a disability' to reduce countable income to zero. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Applied an $880 disability-related medical deduction and an uncapped excess shelter deduction for a head with no SSI/SSDI. Its annual $3,849.60 net income corresponds to roughly $321/mo. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"Included disability-related medical costs and uncapped shelter costs in the calculation. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Claimed income and assets exceed SNAP limits. Assets are only $130, gross income passes California's 200% FPL BBCE screen, and the capped calculation under c_snap_hold_fy2026 gives $877 rounded net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Said the disabled head faces only the net income test, and took the $1,180 medical deduction plus an uncapped $14,837 shelter deduction. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring. It also used the FY2025 $292 maximum."
-us,scenario_023,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Correctly applied California's 200% FPL BBCE gross test, but then took the excess medical deduction and an uncapped shelter deduction as if the household were SNAP-disabled. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring. It also used a $292 maximum instead of $298."
-us,scenario_023,snap,grok-4.7,prompt_ambiguity,age_disability,False,"Correctly passed the BBCE screen and left out the utility allowance, but applied the excess medical deduction and an uncapped shelter deduction. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Left the $8,000 403(b) distribution out of countable income and applied the medical deduction and uncapped shelter for a 'disabled' head who receives no SSI/SSDI. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,inkling,prompt_ambiguity,age_disability,False,"Applied the medical deduction and an uncapped shelter deduction on the basis of disability. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,kimi-k2.6,prompt_ambiguity,age_disability,False,"Exempted the household from the gross test and applied the medical deduction and uncapped shelter because of the disability flag. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring. It also used the FY2025 $292 maximum."
-us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"Gave no SNAP value and no explanation, so there was no parseable answer. The excluded frozen reference is $461.34 for the year. The current engine under c_snap_hold_fy2026 gives $408 on the capped reading."
-us,scenario_023,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Subtracted the traditional 401(k) and IRA contributions from gross income, giving $22,535 instead of $25,443; SNAP counts gross wages. It then applied the 130% FPL gross test instead of California's 200% BBCE screen, which the household passes. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,muse-spark-1.3,prompt_ambiguity,age_disability,False,"Correctly passed the 200% BBCE screen, but applied a $57 disabled medical deduction and uncapped shelter under the disability reading. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring. It also used a $292 maximum."
-us,scenario_023,snap,muse-wonder,prompt_ambiguity,age_disability,False,"Treated the head as SNAP-disabled and used the medical and uncapped shelter deductions to bring net income to zero. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Correctly skipped the medical deduction and capped shelter at $744, but used the FY2025 $204 standard deduction instead of FY2026's $209, which gave net income of about $882. Applying FY2026 parameters to all 12 months follows c_snap_hold_fy2026. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Left out the $8,000 403(b) distribution from countable unearned income and skipped the capped $744 shelter deduction. It then awarded its assumed $258 monthly maximum even though its own net income of $944 implies a $284 contribution. On the capped reading, the current engine with c_snap_hold_fy2026 gives $877 net income and $34/mo ($408/year). This output is excluded from scoring."
-us,scenario_023,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Called income and resources too high without computing anything. California's 200% FPL BBCE screen is passed, assets are $130, and the capped calculation under c_snap_hold_fy2026 gives $877 rounded net income and $34/mo ($408/year) for all 12 months. This output is excluded from scoring."
-us,scenario_023,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model treated the narrative phrase “is disabled” as establishing SSI disability eligibility, despite the prompt directing that unlisted program-status inputs are false. The head fails the SSI aged, blind, or disabled gate, so applying earned-income exclusions, counting the 403(b) distribution, and subtracting income from the federal-plus-California payment standard was inapplicable."
-us,scenario_023,ssi,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It computed AGI correctly and named both credits (~$154 + $60 = ~$214). But it understated the tentative tax as ~$200 when the correct figure is $225.78, then rounded the remainder to $0 instead of reporting the $12.78 left after the $213 in credits."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It projected a 2026 standard deduction of ~$5,880 instead of the $5,706 that California published for 2025. That understated taxable income and gave tax of ~$222 instead of $225.78, so the result after the $153 and $60 credits was $9, not $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left the $8,000 taxable 403(b) distribution and the 401(k)/IRA adjustments out of CA AGI and used the outdated $5,202 standard deduction. It then treated CalEITC as a nonrefundable credit, but CalEITC is refundable and is excluded from tax before refundable credits."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed taxable income is zero after the standard deduction. In fact CA AGI of $22,534 exceeds the $5,706 standard deduction by $16,828, which produces $225.78 of tax before the $213 in nonrefundable credits."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It miscomputed tax on ~$17,000 of taxable income as ~$170 (the correct 1%/2% figure is about $226) and used the 2024 values ($5,540 deduction, $149 credit). It also omitted the $60 renter's credit and invented 'disabled-related considerations' to reach $0, when a $12.78 liability remains."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It computed ~$233 of gross tax and a ~$149 exemption credit, then declared the liability $0 even though its own numbers leave about $84. It also never applied the $60 renter's credit. The correct figure with 2025 parameters is $225.78 − $153 − $60 = $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It estimated ~$200 of tax and a ~$150 exemption credit, then asserted the tax was fully offset, which its own figures contradict. It also did not apply the $60 renter's credit. The $153 exemption credit and $60 renter's credit together reduce the $225.78 tax to $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It projected 2026 indexing for the standard deduction (~$5,880) and exemption credit (~$158) instead of using California's published 2025 values of $5,706 and $153. That pushed the residual down to $1 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,age_disability,False,"It added a second $144 exemption credit for disability, which California does not have; the extra exemption credits apply only to blindness and age 65+. That extra credit wrongly floored the tax at $0. It also used the stale 2024 standard deduction and bracket ($5,540 / $10,756) instead of $5,706 / $11,079."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It estimated $250–$300 of tax, then claimed a ~$149 exemption credit plus nonexistent 'disability-related credits' eliminated it. California grants no disability exemption credit, and the $153 exemption plus $60 renter's credit leave $12.78 of the $225.78 tax."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,other,False,"It computed the tentative tax (~$226) and exemption credit ($153) correctly but omitted the $60 nonrefundable renter's credit. A single filer paying $24,000 rent with AGI under the limit qualifies for it, so the answer should be $12.78, not ~$75."
-us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used AGI of $14,446, which leaves out the $8,000 taxable 403(b) distribution that California includes in AGI. That put taxable income in the 1% bracket and let the exemption credit wipe out the tax. The correct CA AGI of $22,534 yields $225.78 of tax, and $12.78 remains after credits."
-us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It left the $8,000 taxable 403(b) distribution out of CA AGI ($14,446 instead of $22,534) and used the outdated $5,363 standard deduction. With the distribution included, tax is $225.78, which exceeds the $213 in nonrefundable credits."
-us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It projected 2026 indexing: a $5,876 standard deduction, an $11,044 bracket and a $158 exemption credit. California's published 2025 values are $5,706, $11,079 and $153, which give $225.78 − $213 = $12.78 rather than $4.74."
-us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It left out the $130 traditional IRA deduction (AGI $22,665 instead of $22,534) and used the 2024 standard deduction and bracket ($5,540 / $10,756) instead of the 2025 values ($5,706 / $11,079). That overstated tax at $234.94 and gave $20.94 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used an unpublished ~$5,800 standard deduction and a ~$155 exemption credit and rounded tax up to ~$230. The published 2025 values ($5,706 deduction, $11,079 bracket, $153 credit) give $225.78 − $213 = $12.78, not $15."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It claimed the single standard deduction exceeds AGI. California AGI is $22,534 against a $5,706 standard deduction, leaving $16,828 taxable and $225.78 of tax before the $213 in credits."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"It asserted the personal exemption credit fully offsets the tax. Tax on $16,828 of taxable income is $225.78, and the $153 exemption credit plus the $60 renter's credit leave $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its taxable income of $8,846 shows it left the $8,000 taxable 403(b) distribution out of CA AGI. Including it gives $16,828 of taxable income and $225.78 of tax, which the $213 in nonrefundable credits do not fully offset."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave $0 with no computation, which implies it assumed low income produces no CA tax. CA AGI of $22,534 less the $5,706 deduction yields $225.78 of tax, and $12.78 remains after the $153 exemption and $60 renter's credits."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"Its $218 roughly equals the tentative tax with neither nonrefundable credit subtracted. California's $153 personal exemption credit and $60 renter's credit both reduce tax before refundable credits, bringing $225.78 down to $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"It asserted the standard deduction and exemption credits bring tax to $0. The $5,706 deduction leaves $16,828 taxable, and the resulting $225.78 tax exceeds the $213 combined exemption and renter's credits by $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,Its $17 matches using the 2024 $149 exemption credit instead of the 2025 $153 credit ($225.78 − $149 − $60 ≈ $17). With the $153 credit the result is $12.78.
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,other,False,"It claimed the standard deduction, exemption credit and renter's credit eliminate the tax. Tax on $16,828 of taxable income is $225.78, and only $213 of nonrefundable credits apply, leaving $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It treated California's personal exemption as a $138 deduction from income and stated that no nonrefundable credits apply. California provides a $153 personal exemption credit against tax and a $60 nonrefundable renter's credit; it also used the stale $5,363 standard deduction."
-us,scenario_023,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It set CA AGI to $25,443 without excluding the $2,778 traditional 401(k) deferral or the $130 IRA deduction, both of which California conforms to. It also subtracted only the $60 renter's credit and omitted the $153 personal exemption credit."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It asserted the deduction and credits wipe out the tax without computing it. The $5,706 deduction leaves $16,828 taxable and $225.78 of tax, which exceeds the $213 in nonrefundable credits by $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Its $72 approximates the $72.78 left after subtracting only the $153 exemption credit from $225.78, so it omitted the $60 nonrefundable renter's credit that this renter qualifies for. It also cited American Opportunity Credit amounts, which California does not offer."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,age_disability,False,"It added a 2.5% California early-distribution tax on the $8,000 403(b) distribution, but the head is disabled, which is an exception to the early-withdrawal penalty under IRC §72(t)(2)(A)(iii). It also wrongly treated the regular tax as fully eliminated when $12.78 remains after the $213 in credits."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"It assumed the personal exemption and renter's credits fully offset the tax. The $153 + $60 credits fall $12.78 short of the $225.78 tax on $16,828 of taxable income."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It used a projected 2026 standard deduction and exemption credit above California's published 2025 values ($5,706 and $153), which shrank the residual to $4. The 2025 parameters give $225.78 − $213 = $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"It claimed the exemption and renter's credits fully offset the tax. The tax is $225.78 and the credits total $213, so $12.78 remains."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It applied indexed 2026 amounts to the standard deduction and exemption credit instead of California's published 2025 values ($5,706 deduction, $11,079 bracket, $153 credit). That cut the residual to $4 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"It reached the right structure (tax after standard deduction, less the exemption and renter's credits) but rounded the result to $13. The exact computation is $225.78 − $153 − $60 = $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It used projected 2026 indexed deduction, bracket and credit amounts that exceed California's published 2025 values, which left only ~$3. The $5,706 deduction, $11,079 bracket and $153 credit give $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used AGI of $14,446, which leaves out the $8,000 taxable 403(b) distribution. With CA AGI of $22,534, tax is $225.78 and the $213 in credits do not eliminate it."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It applied over-indexed 2026 deduction and credit amounts instead of California's published 2025 values, which produced $6. The $5,706 deduction, $11,079 bracket and $153 credit give $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction produces zero tax before credits. The $5,706 deduction leaves $16,828 taxable and $225.78 of tax, and $12.78 remains after the $213 in nonrefundable credits."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used a projected $158 exemption credit and inflated brackets instead of California's published 2025 $153 credit and $11,079 bracket. That lowered the residual to $9 instead of $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It claimed an itemized medical deduction of ~$7,499 by treating pre-tax employer insurance premiums as deductible. Those premiums are already excluded from wages, and the deductible out-of-pocket costs ($500 medical + $300 premiums) fall below the 7.5%-of-AGI floor. The $5,706 standard deduction therefore applies."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It projected 2026 amounts: a $5,843 deduction, an $11,345 bracket and a $157 credit. California's published 2025 values are $5,706, $11,079 and $153, which give $225.78 − $213 = $12.78 rather than $3."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,other,False,It subtracted only the $60 renter's credit and omitted California's $153 single personal exemption credit. That left $169 instead of $225.78 − $213 = $12.78.
-us,scenario_023,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used AGI of ~$14,446, which leaves out the $8,000 taxable 403(b) distribution. Including it gives $16,828 of taxable income and $225.78 of tax, which exceeds the exemption credit."
-us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no value or explanation for state_income_tax_before_refundable_credits.
-us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,It returned no value or explanation for state_income_tax_before_refundable_credits.
-us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It added the traditional 401(k) and IRA contributions back to CA AGI, but California conforms to both the elective-deferral exclusion and the IRA deduction. It also applied neither the $153 exemption credit nor the $60 renter's credit."
-us,scenario_023,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used the 2024 $149 exemption credit instead of the 2025 $153 credit and overstated the tentative tax at ~$232. The correct tax on $16,828 of taxable income is $225.78, so the answer is $12.78, not $23."
-us,scenario_023,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It used ~$17,000 of taxable income, implying a 2024-level standard deduction, plus the 2024 $149 exemption credit. The 2025 $5,706 deduction and $153 credit give $225.78 − $213 = $12.78 rather than $27."
-us,scenario_023,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It paired the 2024 $5,540 standard deduction and $149 exemption credit with a 2025-level bracket. The 2025 values ($5,706 deduction, $153 credit) give $12.78, not $20."
-us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"It left the 401(k) and IRA adjustments out of AGI ($25,443) and added a nonexistent $144 disability exemption credit, since California grants extra exemption credits only for blindness and age 65+. It also misclassified the renter's credit as refundable; the correct $153 + $60 nonrefundable credits leave $12.78."
-us,scenario_023,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It asserted the tax is zero after the standard deduction and nonrefundable credits. The $5,706 deduction leaves $16,828 taxable and $225.78 of tax, which exceeds the $213 in credits by $12.78."
-us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It lowered earned income by the 401(k) deferral to $14,665 and gave $402. That is above the childless CalEITC maximum of about $300, so this filer cannot get it at any income. The correct phase-out runs on the larger AGI of $22,534 and leaves $95.15."
-us,scenario_023,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"It phased out the childless CalEITC using wages of $17,443 alone. It never used the greater of earned income or AGI ($22,534 with the 403(b) distribution included). At that AGI the credit drops to $95.15, not $180."
-us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It said a filer with no dependent cannot get CalEITC. California pays CalEITC to childless filers aged 18 and older. This 28-year-old qualifies, and the credit after the AGI-based phase-out is $95.15."
-us,scenario_023,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It treated CalEITC as unavailable without qualifying children. The childless CalEITC applies to filers aged 18 and older, so this 28-year-old gets $95.15 after the phase-out based on AGI of $22,534."
-us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Its own phase-out estimate was about $44, but it then raised the answer to $358 based on unrelated 'interactions' with the federal AOC. $358 is above the childless CalEITC maximum. The correct phase-out on AGI of $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It acknowledged the childless CalEITC is small at this income, then gave $1,208. That is about four times the childless maximum, so it used a with-child credit size. The childless schedule phased out on AGI of $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It claimed the childless CalEITC is fully phased out below $17,443 of earnings. The CalEITC income ceiling is $32,901, so the filer is still in the phase-out range at AGI of $22,534 and gets $95.15."
-us,scenario_023,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"It correctly ran the phase-out on AGI of about $22.5k. However, it used a 2026 maximum credit or phase-out slope that overstated the remaining credit, giving $115 instead of $95.15."
-us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It counted the $8,000 taxable 403(b) distribution as investment income and disqualified the filer under the CalEITC investment-income limit. Pension distributions are not investment income for that test. The distribution only raises AGI, which reduces the credit to $95.15 without eliminating it."
-us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It gave $285, close to the childless maximum, as if the filer were on the plateau. It never applied the phase-out on the greater of earnings or AGI ($22,534), which cuts the credit to $95.15."
-us,scenario_023,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It placed the filer on the phase-out slope using earned income of $17,443 only. It ignored that the phase-out uses the higher AGI of $22,534, which includes the 403(b) distribution and leaves $95.15."
-us,scenario_023,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It used made-up figures of $6,576 earned income and $14,446 AGI and treated the filer as near the $300 maximum. Actual earned income is $17,443 and AGI is $22,534. The phase-out on that AGI leaves $95.15."
-us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It modeled CalEITC as 85% of a $503 federal EITC. CalEITC has its own schedule and phase-out and is not a percentage of the federal credit. At AGI of $22,534 the federal childless EITC is zero anyway, while the separate CalEITC schedule gives $95.15."
-us,scenario_023,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It said no CalEITC applies. A childless 28-year-old wage earner with AGI of $22,534 is within the CalEITC income range and gets $95.15 after the phase-out."
-us,scenario_023,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"It invented a linear phase-out from $300 at $15,000 to zero at $33,000 and applied it to AGI of $22,665. The real childless CalEITC schedule starts phasing out at $4,661 of income, and its final phase-out takes the credit from $252 to zero at $32,901. That leaves $95.15 at AGI of $22,534."
-us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It correctly ran the phase-out on AGI of $22,535 but rounded the result to a $100 estimate. The exact CalEITC schedule gives $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It said income is too high for any California refundable credit. The childless CalEITC phase-out reaches $32,901, so at AGI of $22,534 the filer still gets $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It said it applied the AGI phase-out only partially and landed on $148. The full CalEITC phase-out at AGI of $22,534 leaves $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"It estimated CalEITC at $180 without giving a phase-out calculation. The credit is reduced using the greater of earnings or AGI ($22,534), which gives $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It said there were no California refundable credits and missed the childless CalEITC. The filer is 28 with $17,443 of wages and AGI of $22,534, which gives a CalEITC of $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It claimed the filer's income exceeds the CalEITC limit. The limit is $32,901, so at AGI of $22,534 the filer is inside the phase-out range and gets $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"Its $195 implies a phase-out based on wages alone, or barely any phase-out. Using the greater of earnings or AGI ($22,534) reduces the credit to $95.15."
-us,scenario_023,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"It correctly identified the phased-out childless CalEITC but used slightly wrong 2026 phase-out parameters, giving $92 instead of $95.15 at AGI of $22,534."
-us,scenario_023,state_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"It listed AGI of $22,535 but did not apply the phase-out on that AGI in full, giving $164. The correct CalEITC schedule at that AGI gives $95.15."
-us,scenario_023,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It claimed the childless CalEITC is fully gone before AGI of $22,535. The phase-out runs to $32,901, so the filer keeps $95.15."
-us,scenario_023,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"It said the filer fails the childless age test. The childless CalEITC requires age 18 or older, which a 28-year-old meets, so the filer gets $95.15."
-us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,It found no California refundable credit and missed that a childless filer aged 18 or older with wages qualifies for CalEITC. That credit is $95.15 after the AGI-based phase-out.
-us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It answered $0 and missed the childless CalEITC. The filer has $17,443 of wages and AGI of $22,534, which gives $95.15."
-us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It correctly placed the filer in the childless phase-out range, but its 2026 phase-out parameters left too much credit, giving $114 instead of $95.15 at AGI of $22,534."
-us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It set the credit from earned income of $17,443 alone. The phase-out uses the higher AGI of $22,534, which includes the 403(b) distribution and cuts the credit to $95.15."
-us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It figured the credit 'at $17,443 of earned income' and ignored that the phase-out uses the greater of earnings or AGI. AGI is $22,534, which leaves $95.15."
-us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It produced no CalEITC. A childless 28-year-old with $17,443 of wages and AGI of $22,534 is inside the CalEITC range and gets $95.15."
-us,scenario_023,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"It phased the childless CalEITC out too far and gave $25. The actual phase-out at AGI of $22,534 leaves $95.15."
-us,scenario_023,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It correctly ran the phase-out on the higher AGI but used slightly wrong 2026 parameters, giving $106 instead of $95.15."
-us,scenario_023,state_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It said a childless adult at this income gets no CalEITC. The childless phase-out runs to $32,901, so at AGI of $22,534 the credit is $95.15."
-us,scenario_023,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"It used too steep a childless phase-out schedule and gave $28. The actual CalEITC phase-out at AGI of $22,534 leaves $95.15."
-us,scenario_023,state_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,"It said it included the retirement distribution in the phase-out income but still gave $145. The real CalEITC phase-out at AGI of $22,534 reduces the credit to $95.15."
-us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It found no qualifying state refundable credit and missed the childless CalEITC. The filer is 28 with AGI of $22,534 and gets $95.15."
-us,scenario_023,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"It made CalEITC depend on getting the federal EITC. CalEITC has its own schedule that reaches $32,901, so the filer gets $95.15 even though the federal childless EITC is zero at this AGI."
-us,scenario_023,state_refundable_credits,grok-4.6,llm_error,categorical_eligibility,False,"It reasoned that a $0 federal EITC means a $0 CalEITC. CalEITC is computed separately, does not require the federal credit, and its phase-out at AGI of $22,534 leaves $95.15."
-us,scenario_023,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"It correctly ran the phase-out on AGI of $22,535 because AGI is higher than earnings. However, its 2026 phase-out parameters overstated the credit, giving $120 instead of $95.15."
-us,scenario_023,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"It gave roughly the full childless maximum of $300 and applied no phase-out. With AGI of $22,534 the credit drops to $95.15."
-us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model gave no parseable value for state_refundable_credits. The correct output is the $95.15 CalEITC.
-us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model gave no parseable value for state_refundable_credits. The correct output is the $95.15 CalEITC.
-us,scenario_023,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It claimed CalEITC is nonrefundable for filers without children. CalEITC is fully refundable for all eligible filers, including childless ones, so the $95.15 credit counts."
-us,scenario_023,state_refundable_credits,muse-spark-1.3,llm_error,credit_phaseout,False,"It based the credit on earned income of $17,443 and ignored that the phase-out uses the higher AGI of $22,534. Using AGI reduces the credit to $95.15."
-us,scenario_023,state_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"It correctly identified a small phased-out childless CalEITC but rounded it to $100. The exact schedule at AGI of $22,534 gives $95.15."
-us,scenario_023,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It claimed $14,665 of earned income is far above the childless phase-out ceiling. The CalEITC ceiling is $32,901, so the filer gets $95.15 at AGI of $22,534."
-us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It added the $60 renter's credit, which is nonrefundable, and estimated CalEITC at $251 with almost no phase-out. Only CalEITC counts, and phased out on AGI of $22,534 it is $95.15."
-us,scenario_023,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It said no California refundable credit applies and missed the childless CalEITC. The filer is 28 with AGI of $22,534 and gets $95.15."
+us,scenario_023,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"It treated the head as SNAP-disabled because of the disability flag, then took an excess medical deduction and an uncapped shelter deduction to reach about $345/month net. The head receives no SSI or SSDI, so medical costs are not deductible and shelter is capped at $744, which leaves net income of $876.51 and a benefit of $35.20/month. It also submitted 3,936, which does not match its own $2,328 calculation."
+us,scenario_023,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"It applied a medical deduction and an uncapped shelter deduction (with an SUA added) because it took the head to be SNAP-disabled, which brought net income to $0 and gave the $298 maximum. With no SSI or SSDI receipt the household is not elderly/disabled, so the shelter deduction stops at the $744 cap and net income is $876.51, not $0."
+us,scenario_023,snap,claude-fable-5.2,prompt_ambiguity,age_disability,False,"It invented SSI receipt to make the head SNAP-disabled and left the $8,000 of 403(b) distributions out of countable unearned income. SSI is $0 here and retirement distributions count as income, so gross is $2,120/month, the shelter deduction is capped at $744, and net is $876.51. That gives $35.20/month, not the maximum allotment."
+us,scenario_023,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"It applied the federal 130% FPL gross income limit and dragged in irrelevant rent and asset reasoning to deny eligibility. California's broad-based categorical eligibility sets the gross screen at 200% FPL ($2,608/month), which the $2,120 gross passes. After the capped $744 shelter deduction, net income of $876.51 is under the $1,304 net limit, so the benefit is $35.20/month."
+us,scenario_023,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"It said income exceeds the SNAP limits, but $2,120/month gross is under CA's 200% FPL BBCE screen. Net income after the earned-income, $209 standard and $744 capped shelter deductions is $876.51, below the $1,304 net limit, which leaves a positive benefit of $35.20/month."
+us,scenario_023,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"It treated the head as SNAP-disabled and so took a $1,180 medical deduction and an uncapped $14,837 shelter deduction. The head gets no disability benefits, so neither applies; with shelter capped at $744/month, net income is $876.51/month. It then submitted 3,768, an unexplained figure that differs from its own $2,529."
+us,scenario_023,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"It treated the household as disabled, deducted medical costs above $35 and left the shelter deduction uncapped. It also computed the 20% earned-income deduction on annual-to-monthly figures incorrectly ($121 instead of $291). It then raised its own $1,788 result to 2,549 with no basis. The correct path has no medical deduction and a $744 shelter cap, giving net income of $876.51."
+us,scenario_023,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"It claimed $2,120/month exceeds a 200% FPL limit of about $2,510, which contradicts its own figures, since the actual limit is $2,608. It also claimed net income exceeds the net limit once the shelter cap applies. With the $744 cap, net income is $876.51, under the $1,304 limit, and the benefit is $35.20/month rather than zero."
+us,scenario_023,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"It assumed the disability flag removes the shelter cap and allows the medical deduction, which pushed net income to $0. SNAP-disabled status requires receiving disability benefits, which the head does not, so the shelter deduction is capped at $744 and net income is $876.51."
+us,scenario_023,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"It gave a $73 medical deduction and an uncapped $1,224 shelter deduction on the grounds that the head is disabled, ending at $328 net income. Without SSI or SSDI receipt the household is not SNAP elderly/disabled, so it gets no medical deduction and a $744 shelter cap, which yields $876.51 net."
+us,scenario_023,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"It denied eligibility under the 130% FPL gross income test. California's BBCE raises the gross screen to 200% FPL, and the $2,120/month gross passes it. After the capped $744 shelter deduction, net income is $876.51, which qualifies for $35.20/month."
+us,scenario_023,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"It treated the head as SNAP-disabled and applied the medical deduction plus an uncapped shelter deduction, reaching net income of about $346. The head receives no disability benefits, so shelter is capped at $744 and net income is $876.51, which gives $35.20/month."
+us,scenario_023,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"It left the excess shelter deduction uncapped ""because head is disabled,"" which produced about $446 of net income. The head is not SNAP-disabled because they receive no SSI or SSDI, so the deduction is capped at $744 and net income is $876.51."
+us,scenario_023,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"It treated the household as disabled, took an inflated $747 medical deduction and a $1,561 uncapped shelter deduction, and so reached net income of $0. No disability benefits are received, so there is no medical deduction and shelter is capped at $744. Net income is $876.51, not $0."
+us,scenario_023,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"It applied a $98 excess medical deduction and an uncapped $1,239 shelter deduction on the assumption that the head is SNAP-disabled. Without disability-benefit receipt, both are replaced by a $744 capped shelter deduction alone, which gives $876.51 net and $35.20/month."
+us,scenario_023,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"It took the disability flag as SNAP-disabled status and used the medical deduction and uncapped shelter deduction to reach $344 net. The head receives no SSI or SSDI, so the shelter deduction is capped at $744 and net income is $876.51."
+us,scenario_023,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"It applied a medical deduction and an uncapped shelter deduction (rent plus an unlisted SUA) for a disabled head, which zeroed net income and gave the maximum allotment. The household is not SNAP elderly/disabled, so shelter is capped at $744 and net income is $876.51, which gives $35.20/month rather than the maximum."
+us,scenario_023,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Its $3,492 equals 12 × $291, the full one-person maximum. That only follows if net income is $0, which requires treating the head as SNAP-disabled with uncapped deductions. With the $744 shelter cap and no medical deduction, net income is $876.51 and the benefit is $35.20/month."
+us,scenario_023,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"It reduced net income to zero through uncapped shelter and medical deductions ""due to disability."" The head is not SNAP-disabled because they receive no disability benefits, so the shelter deduction is capped at $744 and net income is $876.51."
+us,scenario_023,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"It used uncapped shelter costs and medical deductions for a ""disabled head"" to reach about $300 of net income. Without SSI or SSDI receipt the household gets neither, so net income is $876.51 after the $744 shelter cap and the benefit is $35.20/month."
+us,scenario_023,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"It asserted that income or resources exceed SNAP limits. The $2,120 gross passes CA's 200% FPL BBCE screen, $130 in assets is trivial, and net income of $876.51 after the $744 capped shelter deduction passes the net test, so the benefit is $35.20/month."
+us,scenario_023,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"It applied uncapped shelter expenses ""for a disabled individual,"" which pushed net income below the correct level. The head receives no disability benefits, so the shelter deduction is capped at $744 and net income is $876.51, which gives $35.20/month."
+us,scenario_023,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"It used an uncapped excess shelter deduction for a ""disabled household"" to reduce net income to $0 and award the maximum. SNAP-disabled status requires disability-benefit receipt, which is absent here, so shelter is capped at $744 and net income is $876.51."
+us,scenario_023,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"It gave an uncapped shelter deduction and medical deductions on the basis that the head is disabled. The household is not SNAP elderly/disabled because it has no SSI or SSDI, so only the $744 capped shelter deduction applies and net income is $876.51."
+us,scenario_023,snap,gemini-4-argon,prompt_ambiguity,age_disability,False,"It claimed the disability makes the shelter deduction uncapped, which reduced net income to $0 and gave the $292 maximum. Without disability-benefit receipt, shelter is capped at $744 and net income is $876.51, which gives $35.20/month."
+us,scenario_023,snap,glm-5.2,prompt_ambiguity,age_disability,False,"It exempted the head from the gross test as disabled and took a $1,180 medical deduction and an uncapped $14,837 shelter deduction. The head is not SNAP-disabled, so neither applies; the shelter deduction is capped at $744/month, giving net income of $876.51/month."
+us,scenario_023,snap,glm-5.3,prompt_ambiguity,age_disability,False,"It subtracted a monthly standard deduction from annual income and compared 30% of annual net income to a monthly maximum allotment, then fabricated $871. It also never applied the $744 capped shelter deduction. Done monthly, net income is $876.51 and the benefit is $35.20/month."
+us,scenario_023,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"It assumed income and assets are too high without computing anything. The $2,120 gross passes CA's 200% FPL BBCE screen and $130 in assets passes, and after the earned-income, $209 standard and $744 capped shelter deductions, net income is $876.51, which gives $35.20/month."
+us,scenario_023,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"It returned 0 because benefit receipt was not listed, which ignores the instruction to assume take-up. The household passes CA BBCE and the net income test at $876.51 net, so the benefit is $35.20/month."
+us,scenario_023,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"It treated the household as a disabled SNAP household and used medical and uncapped shelter deductions to zero out net income. The head gets no SSI or SSDI, so the shelter deduction is capped at $744, there is no medical deduction, and net income is $876.51."
+us,scenario_023,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"It applied medical and uncapped shelter deductions because the household is disabled and awarded the $298 maximum. SNAP-disabled status requires disability-benefit receipt, which the head lacks, so shelter is capped at $744 and net income is $876.51."
+us,scenario_023,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"It used allowable medical and uncapped ""disability shelter"" deductions, which lowered net income enough to give about $202/month. The household is not SNAP elderly/disabled, so only the $744 capped shelter deduction applies, net income is $876.51, and the benefit is $35.20/month."
+us,scenario_023,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"It concluded that earnings plus the retirement distribution leave no benefit. It did not apply the $744 excess shelter deduction generated by $2,000/month rent, which, together with the earned-income and standard deductions, brings net income to $876.51 and the benefit to $35.20/month."
+us,scenario_023,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"It applied a $150 standard medical deduction and an uncapped shelter deduction ""for disability,"" which gave $205.80 net income. Without SSI or SSDI receipt the household gets no medical deduction and a $744 shelter cap, so net income is $876.51."
+us,scenario_023,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"It treated the disabled flag as qualifying the household for medical and uncapped shelter deductions, which led to the maximum $298 allotment. The head is not SNAP-disabled, so shelter is capped at $744 and net income is $876.51."
+us,scenario_023,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"It used an uncapped shelter deduction because of the disability to reduce countable income to zero. With no disability-benefit receipt the cap of $744 applies, which gives net income of $876.51 and a benefit of $35.20/month."
+us,scenario_023,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"It took an $880 disability-related medical deduction and an uncapped shelter deduction, ending at $3,849.60 of annual net income. The household is not SNAP elderly/disabled, so there is no medical deduction and shelter is capped at $744/month, which gives net income of $876.51/month."
+us,scenario_023,snap,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"It applied disability-related medical costs and uncapped shelter costs. Without SSI or SSDI receipt the head is not SNAP-disabled, so only a $744 capped shelter deduction applies, net income is $876.51, and the benefit is $35.20/month."
+us,scenario_023,snap,grok-4.3,prompt_ambiguity,age_disability,False,"It claimed income and assets exceed the limits after the housing deduction. The $2,120 gross passes CA's 200% FPL BBCE screen, and after the $744 capped shelter deduction net income is $876.51, under the $1,304 net limit, so the benefit is $35.20/month."
+us,scenario_023,snap,grok-4.5,prompt_ambiguity,age_disability,False,"It treated the household as disabled and took a $1,180 excess medical deduction and an uncapped $14,837 shelter deduction. Neither applies because the head receives no disability benefits; the shelter deduction is capped at $744/month, giving net income of $876.51/month."
+us,scenario_023,snap,grok-4.6,prompt_ambiguity,age_disability,False,"It applied excess medical and uncapped shelter deductions for a ""disabled"" household, which zeroed net income and gave the $292 maximum. The household is not SNAP elderly/disabled, so shelter is capped at $744 and net income is $876.51."
+us,scenario_023,snap,grok-4.7,prompt_ambiguity,age_disability,False,"It used an excess medical deduction on $1,600 of costs and an uncapped shelter deduction to reach about $283 net. Without SSI or SSDI receipt the head is not SNAP-disabled, so the shelter deduction is capped at $744 and net income is $876.51."
+us,scenario_023,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"It dropped the $8,000 of 403(b) distributions from income and, treating the household as disabled, took a medical deduction and an uncapped shelter deduction to reach $0 net. Retirement distributions count as unearned income, and the household is not SNAP-disabled, so shelter is capped at $744 and net income is $876.51."
+us,scenario_023,snap,inkling,prompt_ambiguity,age_disability,False,"It applied a medical deduction ""for the disabled"" and an uncapped shelter deduction to get about $300 net. The head receives no disability benefits, so only the $744 capped shelter deduction applies, net income is $876.51, and the benefit is $35.20/month."
+us,scenario_023,snap,kimi-k2.6,prompt_ambiguity,age_disability,False,"It treated the disabled member as making the shelter deduction uncapped and medical costs deductible, which zeroed net income. SNAP-disabled status requires disability-benefit receipt, so the shelter deduction is capped at $744 and net income is $876.51, which gives $35.20/month."
+us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"No SNAP value or explanation was returned. The correct derivation gives net income of $876.51/month after a $744 capped shelter deduction, for $461.34 a year."
+us,scenario_023,snap,minimax-m3,prompt_ambiguity,age_disability,False,"It subtracted the traditional 401(k) and IRA contributions from SNAP gross income (to get $22,535), then applied the 130% FPL gross test. California's BBCE uses a 200% FPL gross screen, which the $2,120/month gross passes, and net income of $876.51 qualifies for $35.20/month."
+us,scenario_023,snap,muse-spark-1.3,prompt_ambiguity,age_disability,False,"It applied a disabled-member medical deduction and an uncapped excess shelter deduction ""for disabled renter"" to get about $353 net. The head is not SNAP-disabled because they receive no SSI or SSDI, so the shelter deduction is capped at $744 and net income is $876.51."
+us,scenario_023,snap,muse-wonder,prompt_ambiguity,age_disability,False,"It treated the household as disabled and used medical and uncapped shelter deductions to bring net income to 0. Without disability-benefit receipt the shelter deduction is capped at $744 and net income is $876.51, which gives $35.20/month rather than the $298 maximum."
+us,scenario_023,snap,ox-alpha,prompt_ambiguity,age_disability,False,"It correctly applied the $744 shelter cap with no medical deduction, but used the FY2025 standard deduction of $204 instead of FY2026's $209, which left net income at $882 instead of $876.51. It also annualized a flat $298 maximum allotment and missed the higher $304.68 maximum in the last three months, which raises those months to $48.18."
+us,scenario_023,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"It counted only wages ($1,454/month), leaving out the $8,000 of 403(b) distributions, never applied the excess shelter deduction, and used a $258 maximum allotment. It then awarded the full maximum even though its own $944 net income implies a 30% contribution above $258. The correct figures are gross $2,120, a $744 capped shelter deduction, net income of $876.51, and $35.20/month."
+us,scenario_023,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"It asserted that income and resources are too high. The $2,120 gross passes CA's 200% FPL BBCE screen and $130 in assets passes the resource test. After the earned-income, $209 standard and $744 capped shelter deductions, net income of $876.51 qualifies for $35.20/month."
+us,scenario_023,ssi,claude-fable-5.2,llm_error,categorical_eligibility,False,"The model treated the 'is disabled' input as automatically meeting SSI's disability category. The engine sets is_ssi_aged_blind_disabled=False for this 28-year-old head, so SSI is $0 before any income test applies. To reach a positive $3,716.50, it also chose to leave out the $8,000 taxable 403(b) distribution, which counts as unearned income for SSI, and counted only the wages after the $20/$65 exclusions."
+us,scenario_023,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model said 'the head is disabled and eligible for SSI' based only on the generic disability flag. The engine's SSI gate is_ssi_aged_blind_disabled is False for this head, so SSI is $0. Separately, it used the combined federal-plus-California SSP payment level ($1,568.46/month) for the federal ssi output, and it applied the $20 general exclusion twice (inside the $85 earned exclusion and again on unearned income)."
+us,scenario_023,ssi,kimi-k3,parse_contract_failure,missing_output,False,"The model gave no ssi value and no reasoning, so there was nothing to score. The correct derivation gives $0, because the 28-year-old head is not SSI aged, blind, or disabled under the engine (is_ssi_aged_blind_disabled=False)."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It built AGI correctly at about $22,535 but put the tax at about $200 when the correct figure is $225.78. It then said the $153 + $60 credits ($213) eliminated the tax. Its own credit total is smaller than the true tax, so $12.78 remains."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It used a projected 2026 standard deduction of about $5,880 instead of the $5,706 used for 2026. That cut taxable income by about $174 and the tax by about $3.50, so it got $9 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,thresholds_rates,False,"It projected 2026 inflation increases: a standard deduction of about $5,860 and an exemption credit of about $157. The correct amounts are $5,706 and $153. The larger deduction and credit pushed the remainder down to $4 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It left the $8,000 taxable 403(b) distribution out of income and used a wrong $5,202 standard deduction. It also treated CalEITC as a nonrefundable credit that offsets the tax, but CalEITC is refundable and does not belong in this output. The actual nonrefundable credits are the $153 exemption credit and $60 renter's credit, which leave $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed taxable income is zero after the standard deduction. In fact, California AGI of $22,534 (wages after the 401(k) deferral plus the $8,000 403(b) distribution) minus $5,706 leaves $16,828 taxable, with $225.78 of tax."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"It used the outdated $5,540 standard deduction and badly understated the tax: about $17,000 of taxable income gives roughly $229, not $170. It also left out the $60 renter's credit, then declared $0 even though its own figures leave a positive balance after the ~$149 exemption credit."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It computed about $233 of gross tax and subtracted only the ~$149 exemption credit, which leaves about $84. It then asserted $0 without reason and never applied the $60 renter's credit. The correct figure is $225.78 minus $213 = $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It said the ~$150 exemption credit fully offsets about $200 of tax, which is an arithmetic contradiction. The real tax of $225.78 exceeds the combined $153 exemption credit and $60 renter's credit by $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It used projected 2026 amounts: a standard deduction of about $5,880 and an exemption credit of about $158. The correct amounts are $5,706 and $153, so its result shrank to $1 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"It added a second $144 'disabled' exemption credit. California's additional exemption credit applies to blindness, not disability, so only the single $153 personal exemption credit is available. That credit plus the $60 renter's credit leaves $12.78 of the $225.78 tax."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"It said disability-related credits and the exemption credit exceed the tax. California has no disability exemption credit, and the $153 exemption credit plus the $60 renter's credit ($213) is less than the $225.78 tax, leaving $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It subtracted only the ~$153 personal exemption credit and left out the $60 nonrefundable renter's credit. The household pays $24,000 rent and its income is under the limit, so the credit applies. Without it the result was about $73, rounded to $75, instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Its $14,446 AGI leaves out the $8,000 taxable 403(b) distribution, which California taxes. That understated taxable income at $8,906 instead of $16,828, so the exemption credit appeared to wipe out the tax."
+us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used a $14,446 AGI that leaves out the $8,000 taxable 403(b) distribution, and a $5,363 standard deduction. That shrank taxable income to $9,083 and the tax to $91, which the exemption credit erased. The correct tax is $225.78 before credits."
+us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It projected 2026 amounts: a $5,876 standard deduction, a $11,044 bracket, and a $158 exemption credit. The correct values are the 2025 amounts of $5,706, $11,079, and $153, so its result was $4.74 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its $22,665 AGI does not subtract the $130 traditional IRA deduction. It also used the outdated 2023 $5,540 standard deduction and $10,756 bracket, which overstated taxable income and the tax ($234.94 instead of $225.78), giving $20.94."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used estimated amounts: a ~$5,800 deduction, a ~$155 exemption credit, and a rounded ~$230 tax, which is not what its own taxable income produces. The correct calculation is $5,706, $225.78 of tax, and $153, which gives $12.78, not $15."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It said the single standard deduction exceeds AGI, but $5,706 is far below the $22,534 California AGI. That leaves $16,828 taxable and $225.78 of tax."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"It said the personal exemption credit alone fully offsets the tax. The $153 credit covers only part of the $225.78 tax, and even with the $60 renter's credit, $12.78 remains."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its taxable income of $8,846 leaves out the $8,000 taxable 403(b) distribution. That understated the tax at $88 instead of $225.78, so the credits appeared to cover it completely."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation. Its $0 answer means it treated the tax as fully offset, but the $153 exemption credit plus the $60 renter's credit total $213, which leaves $12.78 of the $225.78 tax."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,It reported roughly the tax before any credits ($218) and never subtracted the $153 personal exemption credit or the $60 renter's credit. Both are nonrefundable and reduce this output to $12.78.
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"It said the standard deduction and exemption credits reduce the tax to $0. In fact, $225.78 of tax on $16,828 of taxable income exceeds the $153 exemption credit plus the $60 renter's credit by $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its structure was right (1%/2% brackets less the exemption and $60 renter's credits). However, it overstated the tax or understated the exemption credit by about $4 compared with the 2025 values ($225.78 of tax, $153 credit), which gave $17 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,other,False,"It said the standard deduction, exemption credit, and renter's credit reduce the tax to $0. The $153 + $60 credits total $213, which is less than the $225.78 tax, so $12.78 remains."
+us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It treated the personal exemption as a $138 deduction from income and said no nonrefundable credits apply. California provides the exemption as a $153 credit against tax, plus a $60 renter's credit for this renter, which together reduce $225.78 to $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It did not exclude the $2,778 traditional 401(k) deferral or subtract the $130 IRA deduction, which pushed AGI to $25,443. It also left out the $153 personal exemption credit and subtracted only the $60 renter's credit."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It said the deduction and credits wipe out the tax. California AGI of $22,534 less $5,706 gives $225.78 of tax, and the $213 of nonrefundable credits leaves $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It cited American Opportunity credit amounts as a California offset, but California has no such credit. Its $72 equals $225.78 minus only the $153 exemption credit, so it left out the $60 nonrefundable renter's credit."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"It said regular tax is eliminated even though $225.78 exceeds the $213 of exemption and renter's credits. It then substituted a 2.5% early-distribution additional tax on the $8,000 403(b) distribution, which the facts do not establish. This output is the rate-schedule tax less nonrefundable credits, which is $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"It said the exemption and renter's credits fully offset the tax. They total $213, which is less than the $225.78 tax on $16,828 of taxable income, leaving $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It used a projected 2026 standard deduction and exemption credit instead of the 2025 published amounts ($5,706 and $153). The larger deduction and credit shrank the result to $4 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"It said the exemption and renter's credits fully offset the tax. They total $213, which is less than the $225.78 tax, so $12.78 remains."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It applied an 'indexed' 2026 standard deduction and exemption credit larger than the $5,706 and $153 used for 2026. That understated the remaining tax at $4 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"Its method was correct (tax less the exemption and renter's credits), but it rounded to whole dollars and reported $13. The exact result is $225.78 − $153 − $60 = $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It used projected 2026 indexed amounts for the standard deduction and exemption credit, which exceed the $5,706 deduction and $153 credit used for 2026. That understated the result at $3 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its $14,446 AGI leaves out the $8,000 taxable 403(b) distribution. With that income included, California AGI is $22,534 and the tax of $225.78 exceeds the credits."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It used projected 2026 amounts for the standard deduction and exemption credit instead of $5,706 and $153. That shaved about $7 off the remainder, giving $6 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It said the standard deduction leaves zero tax before credits. The $5,706 deduction against $22,534 of California AGI leaves $16,828 taxable and $225.78 of tax, which $213 of credits reduces to $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used a projected 2026 exemption credit of $158 instead of $153, which together with its rough bracket estimate gave $9. The correct calculation is $225.78 − $153 − $60 = $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It itemized a ~$7,499 medical deduction by counting the $8,089 employer-sponsored insurance premiums. Only the $300 premiums and the out-of-pocket medical costs count, and those fall far short of the 7.5%-of-AGI floor. California therefore uses the $5,706 standard deduction, giving $225.78 of tax, which the $213 of credits does not fully offset."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It estimated 2026 amounts: a $5,843 standard deduction, a $11,345 bracket, and a $157 exemption credit. The correct values are the 2025 amounts of $5,706, $11,079, and $153, so its result was $3 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,It subtracted only the $60 renter's credit and left out the $153 nonrefundable personal exemption credit for a single filer. That left $169 instead of $12.78.
+us,scenario_023,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its ~$14,446 AGI leaves out the $8,000 taxable 403(b) distribution. That shrank taxable income to about $8,720 and the tax to $87, which the exemption credit erased. The correct taxable income is $16,828, with $225.78 of tax."
+us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value for state_income_tax_before_refundable_credits, so there is no answer to score against the $12.78 reference."
+us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It submitted no value for state_income_tax_before_refundable_credits, so there is no answer to score against the $12.78 reference."
+us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It added the 401(k) deferral and IRA deduction back into income, but California follows the federal treatment of both, so California AGI is $22,534, not $25,443. It also applied no nonrefundable credits, leaving out the $153 exemption credit and the $60 renter's credit."
+us,scenario_023,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It overstated the tax at about $232 (its own taxable income of $16,809 gives about $225 under the $11,079 bracket) and used the outdated $149 exemption credit instead of $153. That gave $23 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It used an understated deduction, putting taxable income at about $17,000 and the tax at $236 instead of $225.78, and the outdated $149 exemption credit instead of $153. That gave $27 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used the outdated $5,540 standard deduction (instead of $5,706) and the $149 exemption credit (instead of $153). Both errors inflated the remainder to $20 instead of $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It did not exclude the 401(k) deferral or subtract the IRA deduction, which pushed AGI to $25,443. It also added a nonexistent $144 'disabled' exemption credit. Only the $153 personal exemption credit and the $60 renter's credit apply, leaving $12.78."
+us,scenario_023,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It said the standard deduction and nonrefundable credits reduce the tax to zero. The $213 of credits ($153 exemption + $60 renter's) is less than the $225.78 tax, so $12.78 remains."
+us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model cut earned income to $14,665 by subtracting the 401(k) deferral and estimated $402. That figure is above the childless CalEITC maximum, and the model never phased out on the higher AGI of $22,534. At that AGI the phase-out tail leaves only $95.15."
+us,scenario_023,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"The model phased out the childless CalEITC on earned income of $17,443 alone and got about $180. CalEITC phases out on the greater of earned income or AGI, and AGI here is $22,534 because of the taxable 403(b) distribution. That higher base cuts the credit to $95.15."
+us,scenario_023,state_refundable_credits,claude-fable-5.2,llm_error,credit_phaseout,False,"The model used earned income reduced by the 401(k) deferral ($14,665) as the phase-out base instead of the greater-of rule with AGI of $22,534. It also misread the phase-out tail, landing at $70 instead of the $95.15 the schedule gives at that AGI."
+us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model ruled the filer out of CalEITC for having no qualifying dependent. CalEITC covers childless filers age 18 and older, so this 28-year-old with $17,443 in wages qualifies and receives $95.15 after the AGI-based phase-out."
+us,scenario_023,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model claimed CalEITC does not apply without qualifying children. CalEITC has a childless schedule for filers 18 and older, and phasing out on AGI of $22,534 gives $95.15."
+us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model first estimated a childless CalEITC of about $44. It then inflated the answer to $358 on the basis of unspecified 'credit interactions' that do not exist. The childless CalEITC, phased out on AGI of $22,534, is $95.15, and no other refundable CA credit applies."
+us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model's own reasoning said the childless credit at this income is small, yet it reported $1,208, which is several times the childless CalEITC maximum of roughly $300. The phase-out on AGI of $22,534 leaves $95.15."
+us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model assumed the childless CalEITC is fully phased out below this income. The CalEITC tail runs to an income limit above $30,000, so at AGI of $22,534 a $95.15 credit remains."
+us,scenario_023,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,The model correctly placed the credit in the phase-out tail at AGI of about $22.5k but overestimated the remaining amount at $115. The 2026 tail slope gives $95.15 at that AGI.
+us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model counted the $8,000 taxable 403(b) distribution as investment income and disqualified the filer from CalEITC. Pension and annuity distributions are not disqualifying investment income under the EITC rules. The model also placed the childless phase-out end near $18,591, but the credit extends past $30,000 and is $95.15 at AGI of $22,534."
+us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model gave an amount near the childless CalEITC maximum ($285), which skips the phase-out. Once the phase-out is applied to the greater of earnings or AGI ($22,534), the credit is $95.15."
+us,scenario_023,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"The model phased out on earned income of $17,443 instead of the higher AGI of $22,534, which includes the 403(b) distribution. That overstated the credit at $150 versus $95.15."
+us,scenario_023,state_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model invented an earned income of $6,576, apparently by subtracting insurance premiums and deferrals from wages, and placed the filer near the maximum credit. Earned income is $17,443 and AGI is $22,534, which puts the filer deep in the phase-out tail at $95.15."
+us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model computed CalEITC as 85% of a federal EITC of $503. CalEITC uses its own phase-in and phase-out schedule, not a percentage of the federal credit. The federal childless EITC is zero at AGI of $22,534 anyway, while the CalEITC schedule gives $95.15."
+us,scenario_023,state_refundable_credits,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model asserted that no CalEITC applies. The 28-year-old childless filer with $17,443 in wages qualifies for CalEITC, which phases out on AGI of $22,534 to $95.15."
+us,scenario_023,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"The model applied a single linear phase-out from $300 at $15,000 to zero at $33,000 on AGI of $22,665. The actual CalEITC schedule has a steeper first phase-out followed by a shallower tail, and it yields $95.15 at AGI of $22,534. The model's AGI also left out the $130 traditional IRA deduction."
+us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model correctly phased out on the higher AGI of $22,535 but rounded the tail value to $100. The exact 2026 phase-out computation gives $95.15."
+us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model decided income was too high for any CA refundable credit. The childless CalEITC phase-out ends above $30,000, so at AGI of $22,534 a $95.15 credit remains."
+us,scenario_023,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model identified the AGI-based phase-out but applied too little reduction and got $148. The 2026 childless tail slope at AGI of $22,534 yields $95.15."
+us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model's $180 matches phasing out on earned income of $17,443 only. CalEITC phases out on the greater of earnings or AGI ($22,534), which yields $95.15."
+us,scenario_023,state_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model reported zero CA refundable credits and missed the childless CalEITC entirely. That credit is $95.15 at AGI of $22,534 for this 28-year-old wage earner."
+us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model claimed income exceeds the CalEITC limits. The childless CalEITC income limit is above $30,000, so at AGI of $22,534 the filer still receives $95.15."
+us,scenario_023,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model's $195 understates the phase-out. It is consistent with using earned income or too shallow a slope instead of phasing out on AGI of $22,534, which leaves $95.15."
+us,scenario_023,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"The model applied the childless phase-out but misjudged the tail value at $92. The exact 2026 schedule on AGI of $22,534 gives $95.15."
+us,scenario_023,state_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"The model used the right inputs ($17,443 of earnings and $22,535 of AGI) but overstated the credit at $164. The 2026 tail phase-out on AGI leaves $95.15."
+us,scenario_023,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model believed the childless CalEITC is fully eliminated before AGI of $22,535. The CalEITC tail extends to an income limit above $30,000, so $95.15 remains."
+us,scenario_023,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"The model claimed CalEITC fails without a qualifying child or a childless age test. CalEITC's minimum age for childless filers is 18, so this 28-year-old qualifies and receives $95.15."
+us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model found no refundable CA credit and overlooked the childless CalEITC. That credit is $95.15 for this wage earner at AGI of $22,534."
+us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model found no refundable CA credit and overlooked the childless CalEITC. That credit is $95.15 for this wage earner at AGI of $22,534."
+us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model placed the credit in the phase-out range but overestimated it at $114. Phasing out the 2026 childless schedule on AGI of $22,534 gives $95.15."
+us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model based the credit on earned income of $17,443 and got $194. It ignored that CalEITC phases out on the greater of earnings or AGI ($22,534), which cuts the credit to $95.15."
+us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model estimated $160 from earned income of $17,443 alone. The higher AGI of $22,534, which includes the 403(b) distribution, governs the phase-out and yields $95.15."
+us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model generated no CA refundable credit and missed the childless CalEITC, which is $95.15 at AGI of $22,534."
+us,scenario_023,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"The model over-phased the childless CalEITC down to $25. The 2026 tail at AGI of $22,534 still leaves $95.15."
+us,scenario_023,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,The model correctly phased out on the higher AGI but overstated the remaining credit at $106. The exact schedule gives $95.15.
+us,scenario_023,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"The model concluded that a childless filer at this income gets no CalEITC. The childless phase-out runs past $30,000, so AGI of $22,534 yields $95.15."
+us,scenario_023,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"The model applied too steep a childless phase-out and got $28. The 2026 tail at AGI of $22,534 leaves $95.15."
+us,scenario_023,state_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,"The model included the retirement distribution in the phase-out base but understated the reduction, giving $145. The schedule at AGI of $22,534 yields $95.15."
+us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model found no qualifying CA refundable credit and missed the childless CalEITC. That credit is $95.15 at AGI of $22,534."
+us,scenario_023,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model made CalEITC conditional on federal EITC eligibility. CalEITC has its own income range that extends well beyond the federal childless phase-out, so it pays $95.15 here even though the federal EITC is zero."
+us,scenario_023,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model set CalEITC to zero because the federal EITC is zero. CalEITC is computed on its own schedule regardless of federal EITC eligibility and gives $95.15 at AGI of $22,534."
+us,scenario_023,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"The model correctly used AGI of $22,535 as the phase-out base but overestimated the tail value at $120. The 2026 schedule yields $95.15."
+us,scenario_023,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model awarded roughly the full childless CalEITC maximum of $300 and skipped the phase-out. At AGI of $22,534 the phase-out leaves only $95.15."
+us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so there was nothing to score against the $95.15 CalEITC."
+us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so there was nothing to score against the $95.15 CalEITC."
+us,scenario_023,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model claimed CalEITC is nonrefundable for filers without children. CalEITC is fully refundable for childless filers too, and it is $95.15 here."
+us,scenario_023,state_refundable_credits,muse-spark-1.3,llm_error,credit_phaseout,False,"The model phased out on earned income of $17,443 and estimated $180. CalEITC phases out on the greater of earnings or AGI ($22,534), giving $95.15."
+us,scenario_023,state_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"The model correctly placed the credit in the phase-out range but rounded the estimate to $100. It did not compute the exact 2026 tail value of $95.15 at AGI of $22,534."
+us,scenario_023,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model claimed $14,665 of earnings exceeds the childless CalEITC ceiling. That ceiling is above $30,000, and phasing out on AGI of $22,534 leaves $95.15."
+us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model added a $60 renter's credit as refundable, but California's renter's credit is nonrefundable. It also overstated the CalEITC at $251 instead of the $95.15 that results from phasing out on AGI of $22,534."
+us,scenario_023,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted that no CA refundable credit applies and missed the childless CalEITC, which is $95.15 at AGI of $22,534."
us,scenario_023,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_023,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_025,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own work used a $32,300 standard deduction ($100 too high) and the correct $24,800 10% bracket to get about $7,019. It then submitted $3,325, which has no connection to that calculation, so the final value contradicts its own reasoning."
@@ -2125,224 +2150,231 @@ us,scenario_025,payroll_tax,glm-5.3,llm_error,other,False,"The model's reasoning
us,scenario_025,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model said only the spouse's $62,725 of wages is taxed at 6.2% + 1.45%, which yields $4,798.46. Its $9,571.50 is about double that. This fits applying the combined 15.3% employer-plus-employee FICA rate instead of the 7.65% employee share."
us,scenario_025,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model computed payroll tax on 'annual wages for both spouses' and reported $9,106. The head has no listed wages, and the spouse's pension is not FICA wages. The correct base is only the spouse's $62,725, giving $4,798.48 at 7.65%."
us,scenario_025,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no parseable payroll_tax value and no reasoning. The correct derivation is 7.65% of the spouse's $62,725 wages, or $4,798.48."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used $2,150 exemptions instead of $1,900 each, left out the $332 base amount on the 2.75% bracket, and skipped the $180.60 medical deduction. That gave about $1,576 after the $200 credit, which it then cut to $875 for a nonexistent 'graduated structure below the flat rate'. The correct result is $332 + 2.75% × $64,894.70 − $200 = $1,916.61."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It got the $3,800 of exemptions and the $200 retirement income credit right, but taxed only 2.75% of income above $26,050 and left out the $332 base amount in Ohio's 2026 schedule. It also skipped the $180.60 unreimbursed medical deduction ($7,300 less 7.5% of FAGI), so its result is about $327 low."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It started from federal taxable income after a $28,000 federal standard deduction, but Ohio starts from federal AGI and has no standard deduction. It then used an obsolete 0.501%–5.75% bracket table in place of the 2026 $332 + 2.75%-over-$26,050 schedule, and skipped the $3,800 of exemptions, the $180.60 medical deduction and the $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It gave a round $1,000 guess with no calculation. The correct path is OAGI $94,744.70, less $3,800 of exemptions, taxed at $332 + 2.75% above $26,050 for $2,116.61, then less the $200 retirement credit, for $1,916.61. Its figure is far below even the plain 2.75% amount on the excess."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used $1,850 exemptions instead of $1,900 and an unexplained 'modest amount' in place of the $332 base. It also said no credits apply, so it dropped the $200 retirement income credit that the spouse's $32,200 pension qualifies for, and it skipped the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Its own work found $1,790 of tax on $91,125 without the $332 base amount, and it named the $200 retirement credit. It then submitted $2,426, which no stated step supports. The correct calculation adds the $332 base to 2.75% of $64,894.70 (after the $180.60 medical deduction) and subtracts $200, giving $1,916.61."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used $2,500 exemptions instead of the $1,900 allowed above $80,000 of MAGI, and a $360 base instead of $332. It then took off about $250 of unnamed credits instead of the $200 retirement income credit, and never applied the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It assumed outright that there is no base amount in the 2026 schedule, so it dropped Ohio's $332 base on the 2.75% bracket. It also skipped the $180.60 medical deduction. Its exemptions and $200 credit were right, but the result is about $327 low."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It said Ohio has no deduction for out-of-pocket health premiums, which is wrong. R.C. 5747.01(A)(10) lets the $6,500 of premiums plus $800 of medical costs above 7.5% of FAGI be deducted, giving $180.60. It also used $2,400 exemptions instead of $1,900 each and left the $332 base amount out of the 2.75% bracket."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It left out the $332 base amount and used $2,350 exemptions instead of $1,900. It then took off about $500 of retirement and senior credits, but the retirement credit tops out at $200 and the senior credit needs age 65, which neither spouse has reached."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It taxed only 2.75% of the $65,075 above $26,050, so it missed the $332 base amount in the 2026 Ohio schedule. It also skipped the $180.60 unreimbursed medical deduction, ending about $327 below the correct $1,916.61."
-us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It made up a schedule (2.5% on the first $26,500 and 3.5% above) instead of using Ohio's 0% bracket to $26,050 and $332 + 2.75% above it. It used $1,000 exemptions instead of $1,900, and left out both the $180.60 medical deduction and the $200 retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used an outdated multi-bracket schedule (2.75%/3.226%/3.688%) instead of the 2026 schedule of $332 + 2.75% above $26,050. It took only one $2,400 exemption instead of two at $1,900, and left out the $180.60 medical deduction and the $200 pension-based retirement income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used $2,800 exemptions instead of $1,900 each and taxed only 2.75% above $26,050, without the $332 base amount. It also said no nonrefundable credits apply, so it missed the $200 retirement income credit and the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It applied 3.5% from $46,100 up, but in 2026 everything from $26,050 to $100,000 is taxed at $332 + 2.75%. It took a 5% joint filing credit, which isn't available because the head has no qualifying income. It also used $2,400 exemptions and left out the $200 retirement credit and the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It took the $21,208 employer-sponsored premium out of FAGI, but those premiums were never in wages, so FAGI stays at $94,925.30. That lowered the 7.5% floor and inflated the medical deduction to $1,821 instead of $180.60. It also counted the $50 OTC spending, used $2,400 exemptions instead of $1,900, and left out the $332 base amount."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no calculation. Its $1,695 is consistent with 2.75% of income above $26,050 without the $332 base amount. The correct path is $332 + 2.75% × ($90,944.70 − $26,050) = $2,116.61, less the $200 retirement credit, for $1,916.61."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It charged only 2.75% above $26,050 on about $90,125 of taxable income, which reflects $2,400 exemptions instead of $1,900. It left out the $332 base amount, the $180.60 medical deduction, and the $200 retirement income credit for the spouse's pension."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used $2,700 exemptions instead of the $1,900 allowed above $80,000 of MAGI, and taxed only 2.75% above $26,050, without the $332 base. It also skipped the $200 retirement income credit and the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no calculation, and $1,280.50 matches no correct step. The right result taxes $90,944.70 (OAGI after the $180.60 medical deduction, less $3,800 of exemptions) at $332 + 2.75% above $26,050, then subtracts the $200 retirement credit, for $1,916.61."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It got the $3,800 of exemptions and the $200 retirement credit right, but charged only 2.75% above $26,050 and left out the $332 base amount of Ohio's 2026 bracket. It also skipped the $180.60 unreimbursed medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $1,694.06 is 2.75% × ($94,925 − $26,050) less $200. So it took no personal exemptions (the correct amount is $3,800), no $180.60 medical deduction, and dropped the $332 base amount on the 2.75% bracket."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It said exemptions drop to $0 above $80,000, but they are $1,900 each at this income. It taxed only 2.75% above $26,050, without the $332 base, and never applied the $200 retirement credit or the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"It used the older $360.69 base amount on the 2.75% bracket instead of the 2026 base of about $332. It also skipped the $180.60 unreimbursed medical deduction, which leaves taxable income at $91,125 instead of $90,944.70."
-us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It took a $30,000 federal standard deduction, but Ohio has no standard deduction and starts from federal AGI. It used $2,400 exemptions, applied 2.75% to all taxable income with no $26,050 zero bracket and no $332 base, and left out the $200 retirement credit and the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used an obsolete five-bracket schedule (2.85%–4.396%), a joint filing credit (not available, since the head has no qualifying income), and a $20 exemption credit (only available when MAGI less personal exemptions is $30,000 or less). Its reasoning reached $3,245, yet it submitted $1,260.80. It never used the 2026 schedule of $332 + 2.75% above $26,050 or the $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated a household with $94,925 of wages and pension as low-income with no tax. Ohio taxable income after $3,800 of exemptions and the $180.60 medical deduction is $90,944.70, far above the $26,050 zero bracket. That produces $2,116.61 of tax, or $1,916.61 after the $200 credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It applied a standard deduction, which Ohio doesn't have; Ohio subtracts personal exemptions ($1,900 × 2) from OAGI. It also never applied the $332 + 2.75% schedule above $26,050, so its $1,340 falls far short of the $1,916.61 the schedule gives after the $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used the old $360.69 base amount instead of the 2026 base of about $332. It also put the $50 OTC spending into the unreimbursed medical worksheet, giving a $230.63 deduction instead of $180.60 from $6,500 premiums plus $800 other expenses. The structure was otherwise right."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used $2,400 exemptions instead of $1,900 and taxed only 2.75% above $26,050, without the $332 base. It said there were no nonrefundable credits, so it missed the $200 retirement credit for the $32,200 pension, and it skipped the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $2,150.25 base tax uses the older $360.69 base amount instead of the 2026 base of about $332. It also left out the $180.60 unreimbursed medical deduction (premiums plus other expenses above 7.5% of FAGI), so it taxed $91,125 instead of $90,944.70."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It removed the $32,200 private pension from Ohio income. Ohio taxes private pensions in full and only gives the $200 retirement income credit (not $25) for them. It also used $2,150 exemptions and a joint filing credit the head doesn't qualify for, because the head has no income."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It counted the $21,208 employer-sponsored premium in the medical worksheet. Only the $6,500 out-of-pocket premiums and $800 of other expenses count, for a $180.60 deduction, not $21,388.63. It also used $2,150 exemptions instead of $1,900, though its $332 base and $200 credit were right."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Its $1,590 is 2.75% × ($91,125 − $26,050) − $200, with no $332 base amount on the 2.75% bracket. It also skipped the $180.60 unreimbursed medical deduction that brings taxable income down to $90,944.70."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It got the exemptions and the $200 retirement credit right, but taxed only 2.75% of income above $26,050 and left out the $332 base amount of the 2026 schedule. It also skipped the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It deducted the full $6,500 of premiums, as if the head couldn't get an employer plan. Because the head has employer-sponsored insurance available, the premiums go on line 3 of the unreimbursed medical worksheet with the $800, subject to the 7.5%-of-FAGI floor, for a deduction of only $180.60. Its $332 base, $3,800 of exemptions and $200 credit were right."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It charged only 2.75% above $26,050 and subtracted the $200 credit, so it left out the $332 base amount on Ohio's 2.75% bracket. The correct tax on $90,944.70 is $2,116.61 before the credit and $1,916.61 after it."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed deductions and credits bring Ohio tax to zero. Ohio only allows $3,800 of exemptions and a $180.60 medical deduction here, which leaves $90,944.70 taxable, far above the $26,050 zero bracket. That is $2,116.61 of tax, minus a $200 credit."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used $2,400 exemptions instead of $1,900 and taxed only 2.75% above $26,050, without the $332 base. It said there were no nonrefundable credits, so it missed the $200 retirement income credit for the spouse's pension, and it skipped the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It left out the $332 base amount on the 2.75% bracket. It also took a 5% joint filing credit, which requires both spouses to have qualifying income, and the head has none. It skipped the $180.60 medical deduction as well."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It handled the exemptions, the $200 retirement credit and the joint filing credit correctly, but taxed only 2.75% above $26,050 and left out the $332 base amount of Ohio's 2026 schedule. It also skipped the $180.60 unreimbursed medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 3.5% above $46,100, but the 2026 schedule taxes everything from $26,050 to $100,000 at $332 + 2.75%. It used $2,650 exemptions instead of $1,900, and left out the $200 retirement credit and the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used $2,400 exemptions, taxed only 2.75% above $26,050 without the $332 base, and said no nonrefundable credits apply, so it missed the $200 retirement income credit. It also skipped the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used $2,500 exemptions instead of the $1,900 allowed above $80,000 of MAGI and taxed only 2.75% above $26,050, without the $332 base amount. It also left out the $200 retirement income credit and the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It got the $1,900 exemptions and the $200 retirement credit right, but computed the tax as only 2.75% of the $65,075 excess and left out the $332 base amount of Ohio's 2026 bracket. It also skipped the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It made up senior or age-based adjustments that bring taxable income near zero. Neither spouse is 65, and Ohio taxable income is $90,944.70 after $3,800 of exemptions and the $180.60 medical deduction. That gives $332 + 2.75% above $26,050 = $2,116.61, less the $200 credit."
-us,scenario_025,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used $2,400 exemptions to reach $90,125 and taxed only 2.75% above $26,050, without the $332 base. It never applied the $200 retirement income credit or the $180.60 medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It got the $3,800 of exemptions and the $200 retirement credit right, but charged only 2.75% of income above $26,050 and left out the $332 base amount of the 2026 Ohio schedule. It also skipped the $180.60 unreimbursed medical deduction."
-us,scenario_025,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It took no personal exemptions (the correct amount is $1,900 × 2) and no $180.60 medical deduction, and taxed only 2.75% above $26,050, without the $332 base. It looked only at the joint filing and senior credits and missed the $200 retirement income credit for the $32,200 pension."
-us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It gave no calculation, and its $2,651 is higher than any correct application of the 2026 Ohio schedule. Taxing $90,944.70 at $332 + 2.75% above $26,050 gives $2,116.61, and the $200 retirement income credit brings that to $1,916.61."
-us,scenario_025,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It took a $9,300 standard deduction, which Ohio doesn't have. It taxed the first $26,050 at 2.75% when that bracket is 0%, and used 3.5% above it instead of $332 + 2.75% up to $100,000. It also claimed a senior credit for a 61-year-old (it requires age 65) and left out the $200 retirement credit and the $3,800 of exemptions."
-us,scenario_026,child1_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It computed MAGI of $82,301, below the 216% FPL limit (211% plus the 5% disregard), but assigned that band to CHIP. In NC that band is Medicaid coverage for older children (the OLDER_CHILD category), so the child is Medicaid-eligible and therefore excluded from CHIP."
-us,scenario_026,child1_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It understated household income at $61,093; wages minus traditional 401(k) and IRA contributions give about $82,301. It also labeled the 211% FPL threshold as NC Health Choice/CHIP, but that program was folded into Medicaid. An 11-year-old under that limit is Medicaid-eligible (OLDER_CHILD), which rules out CHIP."
-us,scenario_026,child1_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"It set NC Medicaid for an 11-year-old at 133% FPL and treated NC Health Choice (211% FPL) as a separate CHIP tier. NC now covers children aged 6-18 up to 211% FPL through Medicaid (OLDER_CHILD), so this child is Medicaid-eligible and not CHIP-eligible."
-us,scenario_026,child1_chip_eligible,grok-4.7,llm_error,categorical_eligibility,False,"It placed the child above a lower Medicaid limit and under a separate NC Health Choice CHIP limit of 211% FPL. It missed that NC's 211% coverage for school-age children is Medicaid (OLDER_CHILD), and Medicaid eligibility excludes the child from CHIP."
-us,scenario_026,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for child1_chip_eligible. The correct derivation finds the 11-year-old Medicaid-eligible under NC's OLDER_CHILD category, which makes CHIP eligibility 0."
-us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"Applied a 133% FPL limit (138% with the disregard) to NC children aged 6–18. NC's older-child Medicaid limit is 211% FPL (216% with the disregard). The model's own MAGI of about $82,300 is 2.13× FPL, which is under that limit."
-us,scenario_026,child1_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"Correctly put income at about 213% FPL but compared it with a 133% FPL limit. NC's Medicaid older-child limit is 211% FPL, or 216% with the 5% disregard, so 2.13× FPL is eligible."
-us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"Used a 200% FPL limit and a 5-person FPL of $27,465, far below the actual guideline of about $38,700. NC's older-child limit is 211% FPL plus the 5% disregard. The model also wrongly treated the child's employer-sponsored insurance as disqualifying, though Medicaid has no uninsured requirement."
-us,scenario_026,child1_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"Said income was 'well above' the child Medicaid limit, which fits a 133–138% FPL federal-minimum limit. NC's Medicaid older-child limit is 211% FPL (216% with the disregard), and the household's MAGI is 2.13× FPL, so Child 1 is eligible."
-us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"Applied a 138% FPL limit (about $51,957) to NC children aged 6–18. NC Medicaid covers this age group up to 211% FPL plus the 5% disregard. The household's MAGI of about $82,300 (2.13× FPL) is within that limit."
-us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"Applied a 138% FPL limit and miscalculated it as about $45,000 for five people. NC's older-child Medicaid limit is 211% FPL (216% with the disregard), and MAGI of 2.13× FPL qualifies."
-us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Used the right 211% FPL limit but overstated income at about 260% FPL. MAGI is the spouse's $85,209 in wages, which already include overtime, minus the $2,778 traditional 401k and $130 IRA contributions. That is about $82,300, or 2.13× FPL, which is within the limit."
-us,scenario_026,child1_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Put income at about 219% FPL, which is what an older 5-person FPL of about $37,650 gives. Against the 2026 guideline, MAGI is 2.13× FPL, below NC's 211% older-child limit plus the 5% disregard (216%)."
-us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"Treated the child's employer-sponsored insurance as disqualifying for Medicaid. Medicaid has no no-other-coverage or uninsured requirement; only CHIP does. On income alone, MAGI of 2.13× FPL is within NC's 216% older-child limit."
-us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Added the $7,746 overtime on top of $85,209 in wages that already include it, and skipped the 401k and IRA deductions. The model also miscalculated 211% FPL for five as $67,000; it is about $81,600, or $83,500 with the disregard. Correct MAGI of about $82,300 (2.13× FPL) is eligible."
-us,scenario_026,child1_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Compared income of about $82,000 with a 133% FPL limit. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, and $82,000 (2.13× FPL) falls under it."
-us,scenario_026,child1_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Put income at about $93,000 by adding overtime on top of wages that already include it, and skipped the 401k deduction. The model also left out the 5% disregard. Correct MAGI of about $82,300 is 2.13× FPL, under NC's 216% effective limit."
-us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"Correctly found MAGI of $82,301 but compared it with a 133% FPL limit ($51,205). NC's Medicaid older-child limit is 211% FPL (216% with the disregard), so 2.13× FPL is eligible."
-us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Correctly found MAGI of $82,301 (2.13× FPL) but said it exceeds NC's child Medicaid limit. Since Health Choice merged into Medicaid, NC's older-child limit has been 211% FPL plus the 5% disregard, which covers this income."
-us,scenario_026,child1_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Compared gross wages of $85,209 with a 210% limit without subtracting the traditional 401k and IRA contributions and without adding the 5% disregard. MAGI of about $82,300 is 2.13× FPL, within the 216% effective limit."
-us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Applied a 147% FPL limit to NC children aged 6–18. The limit is 211% FPL (216% with the disregard). Even the model's own understated MAGI of $61,093, which wrongly subtracts the head's ESI premiums, is far below that limit."
-us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Said income was above the child Medicaid threshold without showing the limit, which fits a 133–138% FPL federal minimum. NC's older-child limit is 211% FPL plus the 5% disregard, and MAGI is 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Applied a 133% FPL Medicaid limit and moved the child to CHIP. NC folded its 6–18 CHIP group (Health Choice) into Medicaid, which now covers up to 211% FPL plus the 5% disregard. MAGI of $82,301 (2.13× FPL) is Medicaid-eligible."
-us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied a 133% FPL limit. The model also understated MAGI as $61,093 by subtracting the head's ESI premiums, which are not wage deductions here. NC's older-child limit is 211% FPL plus the 5% disregard, and actual MAGI of 2.13× FPL qualifies."
-us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Said Child 1 is ineligible with no reasoning, which fits a 133–138% FPL child limit. NC's Medicaid older-child limit is 211% FPL (216% with the disregard), and the household's 2.13× FPL MAGI qualifies."
-us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"Said income exceeds NC's child Medicaid limit, which understates that limit. The limit for ages 6–18 is 211% FPL plus the 5% disregard, and MAGI of about $82,300 is 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"Said income exceeds the limit, which fits applying the federal-minimum 133% FPL instead of NC's 211% (216% with the disregard) older-child Medicaid limit. MAGI of 2.13× FPL qualifies."
-us,scenario_026,child1_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"Said household income exceeds NC's child Medicaid limit. NC covers children aged 6–18 up to 211% FPL plus the 5% disregard, which covers MAGI of 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,gemini-4-argon,llm_error,thresholds_rates,False,"Said MAGI is above NC's child Medicaid limit. Since Health Choice merged into Medicaid, the older-child limit has been 211% FPL plus the 5% disregard, and MAGI of 2.13× FPL falls under it."
-us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,"Treated the child's employer-sponsored insurance as disqualifying for Medicaid. Medicaid has no uninsured requirement; only CHIP does. With MAGI at 2.13× FPL, under NC's 216% older-child limit, Child 1 is eligible."
-us,scenario_026,child1_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"Compared gross wages of $85,209 with the 211% limit without subtracting the $2,778 traditional 401k and $130 IRA contributions and without the 5% disregard. MAGI of about $82,300 is 2.13× FPL, under the 216% effective limit."
-us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"Said earnings are above the child Medicaid threshold, which understates NC's limit for ages 6–18. That limit is 211% FPL plus the 5% disregard, and MAGI is 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"Called the household ineligible based on its wage level without checking it against NC's child limit. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, which covers MAGI of 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"Said income exceeds the combined Medicaid/CHIP limit for school-age children. In NC, that combined limit is the Medicaid older-child limit of 211% FPL plus the 5% disregard, and MAGI of 2.13× FPL is under it."
-us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Estimated NC's child Medicaid threshold too low. For ages 6–18 it is 211% FPL plus the 5% disregard, and MAGI of about $82,300 (2.13× FPL) qualifies."
-us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"Said MAGI exceeds NC's Medicaid limit for an 11-year-old. That limit is 211% FPL plus the 5% disregard, and MAGI is 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"Said income is above NC's child Medicaid limit, which understates that limit. The older-child limit is 211% FPL (216% with the disregard), and MAGI of 2.13× FPL qualifies."
-us,scenario_026,child1_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,Applied an age-11 limit lower than NC's actual Medicaid older-child limit of 211% FPL plus the 5% disregard. The household's MAGI is 2.13× FPL and qualifies.
-us,scenario_026,child1_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"Said income exceeds the child Medicaid limit. NC's limit for ages 6–18 is 211% FPL plus the 5% disregard, which covers the household's 2.13× FPL MAGI."
-us,scenario_026,child1_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"Correctly found MAGI of $82,301 but said it exceeds NC's child limit for five people. That limit is 211% FPL plus the 5% disregard (about 216%), and $82,301 is 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,gpt-6.1-terra,llm_error,thresholds_rates,False,"Said the household's income is too high for children's Medicaid. NC's older-child limit is 211% FPL plus the 5% disregard, and MAGI of 2.13× FPL qualifies."
-us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"Treated the child's employer-sponsored insurance as disqualifying, though Medicaid has no uninsured requirement. The model also understated NC's older-child income limit, which is 211% FPL plus the 5% disregard and covers MAGI of 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"Applied a 133% FPL limit to children aged 6–18. NC's Medicaid older-child limit is 211% FPL (216% with the disregard), and MAGI of 2.13× FPL qualifies."
-us,scenario_026,child1_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"Applied a 133% FPL limit (about $51,000). NC's older-child Medicaid limit is 211% FPL plus the 5% disregard. Even the model's own MAGI of $74,555 is under that limit, and the actual MAGI of 2.13× FPL qualifies."
-us,scenario_026,child1_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"Applied a 138% FPL limit to NC children aged 6–18. The actual Medicaid limit is 211% FPL plus the 5% disregard, which covers MAGI of 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"Applied a 133% FPL limit and overstated income at about 240% FPL. NC's older-child limit is 211% FPL plus the 5% disregard, and actual MAGI (wages minus 401k and IRA contributions) is 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"Limited Medicaid for ages 6–18 to 133% FPL. NC covers these children in Medicaid up to 211% FPL plus the 5% disregard, which covers MAGI of 2.13× FPL. The uninsured requirement the model cited applies to CHIP, not Medicaid."
-us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for child1_medicaid_eligible, so no answer was recorded. The correct answer is eligible: MAGI is 2.13× FPL, within NC's older-child Medicaid limit."
-us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"Applied a 133% FPL limit (138% with the disregard) to children aged 6–18, and overstated income at about 226% FPL using an older FPL. NC's limit is 211% FPL plus the 5% disregard, and MAGI is 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"Said income is 'well above' NC's children's Medicaid limits, which fits a 133–138% FPL limit. The actual older-child limit is 211% FPL plus the 5% disregard, and MAGI is 2.13× FPL."
-us,scenario_026,child1_medicaid_eligible,muse-spark-1.3,llm_error,thresholds_rates,False,"Overstated MAGI at about 221% FPL instead of 2.13× FPL by using an outdated, lower FPL, and left out the 5% disregard that raises the 211% limit to 216%. The model also wrongly treated employer-sponsored insurance as a bar to Medicaid."
-us,scenario_026,child1_medicaid_eligible,muse-wonder,llm_error,thresholds_rates,False,"Said MAGI is above NC's child Medicaid limit. That limit is 211% FPL plus the 5% disregard, and the household's MAGI of 2.13× FPL falls under it."
-us,scenario_026,child1_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"Applied a 133% FPL limit to NC children aged 6–18. The actual limit is 211% FPL plus the 5% disregard, so even the model's own estimate of about 198% FPL qualifies; the actual MAGI of 2.13× FPL qualifies too."
-us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"Correctly found income within the child limit but then disqualified Child 1 for having employer-sponsored insurance. Medicaid has no uninsured or no-other-coverage requirement, so a child at 2.13× FPL is eligible."
-us,scenario_026,child1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"Said income is too high for child Medicaid without citing a limit. NC's older-child limit is 211% FPL plus the 5% disregard, and MAGI of 2.13× FPL qualifies."
-us,scenario_026,child2_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It compared MAGI of $82,301 to a 216% FPL limit and labeled that the NC CHIP limit. That band is the NC Medicaid limit for children aged 6-18. The child qualifies for Medicaid under the older-child category, and Medicaid eligibility excludes CHIP, so the answer is 0, not 1."
-us,scenario_026,child2_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It found income of $61,093 below 211% FPL and treated that as NC Health Choice (CHIP) eligibility. At that income an 11-year-old in NC is Medicaid-eligible under the older-child category, since NC Health Choice is now part of Medicaid. A Medicaid-eligible child is not CHIP-eligible."
-us,scenario_026,child2_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"It used the federal minimum of 133% FPL as the Medicaid ceiling for school-age children and put the 133-211% band in CHIP. NC Medicaid's older-child category covers an 11-year-old across that band, so at MAGI of $74,555 the child is Medicaid-eligible. Medicaid eligibility excludes CHIP."
-us,scenario_026,child2_chip_eligible,grok-4.7,llm_error,categorical_eligibility,False,"It placed household MAGI in a CHIP band above Medicaid and at or below 211% FPL, assuming NC Medicaid for an 11-year-old stops at a lower limit. The engine's NC Medicaid older-child category covers this child at this income. Because the child is Medicaid-eligible, the child is not CHIP-eligible."
-us,scenario_026,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for child2_chip_eligible, so there is no substantive answer to grade. The correct answer is 0: the child is Medicaid-eligible under NC's older-child category, which excludes CHIP."
-us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"It got MAGI right at about $82,301 but divided by a 2025-level FPL (~$37,600-$37,900), which gave 219% instead of the 2.13x ratio under the 2026 guideline. It also compared that ratio to 211% without the 5-point MAGI disregard, even though NC's older-child Medicaid limit is effectively 216%."
-us,scenario_026,child2_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"It computed the right ratio (~213% FPL) but used the old 133% FPL Medicaid limit for ages 6-18. Since NC Health Choice merged into Medicaid in 2023, children 6-18 get Medicaid up to 211% FPL (216% with the disregard), so 213% qualifies."
-us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"It treated employer-sponsored coverage as disqualifying, but ESI does not bar a child from Medicaid. It also wrongly assumed two earners; only the spouse has wages. Actual MAGI is $82,301, or 2.13x FPL, which is under NC's 216% older-child Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"It said income was 'well above' the child limit without doing the math. MAGI of $82,301 is 2.13x the 2026 FPL for five, which is under NC's 211%+5% (216%) Medicaid limit for ages 6-18. Its answer fits applying the old 133% limit."
-us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the 2024 FPL of $36,580 for five, which pushed the ratio above 211%. It then settled on a 133% FPL limit for ages 6-18, but after the 2023 Health Choice merger NC Medicaid covers these children up to 216%, well above the actual 2.13x ratio."
-us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It used ~$85,000 as MAGI without subtracting the $2,778 traditional 401(k) and $130 IRA contributions, and it used the outdated $36,580 FPL. It also treated the 211% tier as separate CHIP rather than Medicaid and left out the 5-point disregard. The correct figures are 2.13x FPL against a 216% Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"It put income at ~260% FPL, but MAGI of $82,301 is only 2.13x the 2026 FPL for a five-person household. That is within NC's 216% Medicaid limit for children aged 6-18."
-us,scenario_026,child2_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"It got about 219% FPL by dividing by the 2025 guideline. Against the 2026 FPL for five, MAGI is 2.13x, which is under NC's older-child Medicaid limit of 211% plus the 5-point disregard."
-us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"It made being uninsured a condition of Medicaid eligibility, but Medicaid has no such rule and having ESI does not disqualify a child. With MAGI at 2.13x FPL, under NC's 216% older-child limit, Child 2 qualifies."
-us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It had the right ~216% limit but turned it into about $67,000, when 216% of the 2026 FPL for five is roughly $83,500. It also counted overtime on top of gross wages even though gross wages already include it. MAGI of $82,301 (2.13x FPL) is under the limit."
-us,scenario_026,child2_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It used the old 133% FPL limit for ages 6-18. Since the 2023 Health Choice merger, NC Medicaid covers these children up to 211% FPL (216% with the disregard), and $82k is 2.13x FPL, so the child qualifies."
-us,scenario_026,child2_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It compared income to a bare 210% FPL limit and left out the 5-point MAGI disregard. NC's effective older-child Medicaid limit is 216%, and the household's 2.13x FPL is under it."
-us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"It got MAGI right at $82,301 but tested it against the old 133% FPL limit. After the 2023 Health Choice merger, NC Medicaid covers children 6-18 up to 211% FPL plus the 5-point disregard, and 2.13x FPL is within that."
-us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It got MAGI right at $82,301, which is 2.13x the 2026 FPL for five, but said it exceeded the child limit without naming one. NC's Medicaid limit for ages 6-18 is 216% with the disregard, so this income qualifies. Its answer fits using the old 133% limit."
-us,scenario_026,child2_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It tested gross wages of $85,209 instead of MAGI, which is $82,301 after the traditional 401(k) and IRA deductions. It also used a 210% limit without the 5-point disregard. MAGI is 2.13x FPL against an effective 216% Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the old 133% FPL limit for ages 6-18, with a stale FPL to match. NC Medicaid now covers these children up to 216% FPL with the disregard, and even its own understated $61,093 MAGI is far below that."
-us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It said income was over the limit without doing any math. MAGI of $82,301 is 2.13x the 2026 FPL for five, under NC's 216% Medicaid limit for ages 6-18, so its answer fits using the old 133% limit."
-us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It treated 133% FPL as the Medicaid limit for older children and put the 133-211% range under separate CHIP. NC merged Health Choice into Medicaid in 2023, so a child at 2.13x FPL is Medicaid-eligible."
-us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used the old 133% FPL limit for ages 6-18. NC Medicaid covers these children up to 211% FPL plus the 5-point disregard, which is well above both its $61,093 estimate and the actual 2.13x FPL MAGI."
-us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no income test at all. MAGI of $82,301 is 2.13x the 2026 FPL, under NC's 216% older-child Medicaid limit, and its No fits using the old 133% limit."
-us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"It said income exceeds NC's limit without giving numbers. MAGI is 2.13x FPL, under the 211%+5% Medicaid limit for ages 6-18 that has applied since Health Choice merged into Medicaid."
-us,scenario_026,child2_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"It said income exceeds the limit without doing any calculation. MAGI is $82,301, or 2.13x the 2026 FPL for five, which is within NC's 216% older-child Medicaid limit, so its answer fits using the old 133% limit."
-us,scenario_026,child2_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,It said income exceeds NC's child Medicaid limit without numbers. The 2.13x FPL MAGI ratio is under the 216% limit for ages 6-18 that applies after the Health Choice merger.
-us,scenario_026,child2_medicaid_eligible,gemini-4-argon,llm_error,thresholds_rates,False,"It said MAGI is over the NC child limit without calculating it. MAGI of $82,301 is 2.13x FPL, under NC's older-child Medicaid limit of 211% plus the 5-point disregard."
-us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"It treated NC Medicaid for ages 6-18 as capped at 133%, with Health Choice as a separate CHIP up to 210%, even though that program merged into Medicaid in 2023. It also used gross wages instead of MAGI and left out the 5-point disregard. The correct figures are 2.13x FPL against a 216% Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"It tested gross wages of $85,209 without subtracting the traditional 401(k) and IRA contributions, and it used 211% without the 5-point disregard. MAGI of $82,301 is 2.13x FPL, under the effective 216% limit."
-us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"It said earnings were above the threshold without doing the math. MAGI is 2.13x the 2026 FPL for five, under NC's 216% older-child Medicaid limit, so its answer fits using the old 133% limit."
-us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"It denied eligibility based only on the income level, with no threshold. MAGI of $82,301 (2.13x FPL) is within NC's Medicaid limit of 211% plus the 5-point disregard for children 6-18."
-us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"It said income was 'well above' the NC child limit, but MAGI after the 401(k) and IRA deductions is $82,301, or 2.13x FPL. That is under the 216% Medicaid limit for ages 6-18, so its answer fits using the old 133% limit."
-us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"It estimated the child limit too low. NC Medicaid covers children 6-18 up to 211% FPL plus the 5-point disregard, and the household's MAGI is 2.13x FPL, so the child qualifies."
-us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"It said MAGI exceeds the limit for an 11-year-old without numbers. MAGI of $82,301 is 2.13x the 2026 FPL, under NC's 216% older-child Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It said income was over the limit without calculating it. The 2.13x FPL MAGI ratio is under NC's Medicaid limit for ages 6-18 (211% plus the 5-point disregard), so its answer fits using the old 133% limit."
-us,scenario_026,child2_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"It applied too low an age-specific limit. Since the 2023 Health Choice merger, NC Medicaid covers 11-year-olds up to 216% FPL with the disregard, and MAGI here is 2.13x FPL."
-us,scenario_026,child2_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"It said income exceeds the limit without doing any math. MAGI of $82,301 is 2.13x FPL, within NC's 216% older-child Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"It got MAGI right at $82,301 but put NC's limit for five below it. That MAGI is 2.13x the 2026 FPL, and NC's limit for ages 6-18 is 211% plus the 5-point disregard (about $83,500), so Child 2 qualifies."
-us,scenario_026,child2_medicaid_eligible,gpt-6.1-terra,llm_error,thresholds_rates,False,"It denied income-based children's Medicaid without a threshold. MAGI is 2.13x FPL, which is under NC's 216% Medicaid limit for children 6-18, so its answer fits using the old 133% limit."
-us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"It gave the income limit and ESI as reasons, but ESI does not bar Medicaid. MAGI at 2.13x FPL is under NC's 216% older-child limit."
-us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"It used the old 133% FPL limit for children 6-18. NC Medicaid now covers them up to 211% FPL plus the 5-point disregard, and MAGI here is 2.13x FPL."
-us,scenario_026,child2_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"It understated MAGI at $74,555 and then used the old 133% FPL limit. The correct limit for ages 6-18 is 216% with the disregard, and MAGI is $82,301, or 2.13x FPL."
-us,scenario_026,child2_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"It said MAGI exceeds the 6-18 limit without numbers. MAGI of 2.13x FPL is under NC's 216% Medicaid limit for that age group, so its answer fits using the old 133% limit."
-us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"It overstated income at ~240% FPL, against an actual 2.13x, and used the old 133% limit. NC Medicaid for children 6-18 runs to 216% with the disregard, so the child qualifies."
-us,scenario_026,child2_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used the old 133% FPL limit for ages 6-18 and treated ESI as disqualifying. NC Medicaid covers these children up to 216% FPL whether or not they have ESI, and MAGI here is 2.13x FPL."
-us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for child2_medicaid_eligible, so there is no answer to grade."
-us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"It used 133%/138% FPL as the limit for ages 6-18 and overstated income at ~226% FPL. MAGI is 2.13x the 2026 FPL, and since the Health Choice merger NC Medicaid covers these children up to 216%."
-us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"It said income was 'well above' the limit without numbers. MAGI of $82,301 is 2.13x FPL, within NC's 216% older-child Medicaid limit, so its answer fits using the old 133% limit."
-us,scenario_026,child2_medicaid_eligible,muse-spark-1.3,llm_error,thresholds_rates,False,"It overstated income at ~221% FPL with an outdated guideline, compared it to 211% without the 5-point disregard, and treated ESI as a bar. MAGI is 2.13x the 2026 FPL, under the effective 216% limit, and ESI does not disqualify."
-us,scenario_026,child2_medicaid_eligible,muse-wonder,llm_error,thresholds_rates,False,It said MAGI exceeds the limit without calculating it. MAGI of 2.13x FPL is under NC's Medicaid limit for ages 6-18 (211% plus the 5-point disregard).
-us,scenario_026,child2_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"It put the 6-18 group at a 133% limit, the rule before the merger. Even its own ~198% FPL estimate is under NC's current 216% Medicaid limit, and the actual ratio is 2.13x FPL."
-us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the old 133% limit for ages 6-18 with a too-low FPL of $33,700 for five, and it counted gross wages instead of MAGI. MAGI of $82,301 is 2.13x the 2026 FPL, under NC's 216% Medicaid limit."
-us,scenario_026,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"It called income too high without stating a threshold. MAGI is 2.13x FPL, within NC's older-child Medicaid limit of 211% plus the 5-point disregard, so its answer fits using the old 133% limit."
-us,scenario_026,child3_chip_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a 133%-211% FPL CHIP band for NC children aged 6-18, which assumes Medicaid for older children stops at 133% FPL. At its own 174% FPL figure, Child 3 falls under North Carolina's OLDER_CHILD Medicaid category. Medicaid eligibility excludes CHIP, so the answer is 0."
-us,scenario_026,child3_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It checked only whether $82,301 MAGI was under a 216% FPL 'CHIP limit' and never tested Medicaid first. Child 3 is Medicaid-eligible under the NC OLDER_CHILD category, and that Medicaid eligibility bars CHIP no matter where income falls against the CHIP ceiling."
-us,scenario_026,child3_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"It concluded CHIP eligibility because $61,093 is under 211% FPL, treating that ceiling as a CHIP limit rather than checking Medicaid. At that income Child 3 qualifies for NC Medicaid under the OLDER_CHILD category, and Medicaid-eligible children are excluded from CHIP."
-us,scenario_026,child3_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"It checked only age, Medicare status and a CHIP income range and skipped the Medicaid screen that comes before CHIP. Child 3 meets North Carolina's OLDER_CHILD Medicaid category, which makes the child ineligible for CHIP."
-us,scenario_026,child3_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"It treated 133% FPL as the top of Medicaid for a 9-year-old and put its $74,555 MAGI in a 133%-211% CHIP band. North Carolina's OLDER_CHILD Medicaid category covers Child 3 at this income, and Medicaid eligibility excludes CHIP."
-us,scenario_026,child3_chip_eligible,grok-4.7,llm_error,thresholds_rates,False,"It asserted that household MAGI was 'above Medicaid' and at or below 211% FPL, which puts the child in a separate CHIP tier. In fact Child 3 meets North Carolina's OLDER_CHILD Medicaid category, so the child is Medicaid-eligible and therefore not CHIP-eligible."
-us,scenario_026,child3_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"It stated outright that Child 3 is 'not Medicaid-eligible', and it measured income against a 6-person FPL even though the household has 5 people. Child 3 is Medicaid-eligible under North Carolina's OLDER_CHILD category, which bars CHIP, so 216% NC Health Choice limit is irrelevant."
-us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model applied a 138% FPL Medicaid limit for 6-18-year-olds and sent incomes up to 211% FPL to a separate CHIP. NC merged NC Health Choice into Medicaid, so older children are Medicaid-eligible up to 211% FPL plus the 5% disregard. The model's own MAGI of about $82,301 is 2.13x FPL, which qualifies."
-us,scenario_026,child3_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"The model computed income correctly at about 213% FPL but compared it with an outdated 133% FPL children's Medicaid limit. NC's current Medicaid limit for ages 6-18 is 211% FPL plus the 5% MAGI disregard (216%), which 213% clears."
-us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model stated a 207% FPL limit but converted it to about $56,232 for a family of five, when 207-216% FPL is roughly $80,000-$83,000. It also never subtracted the $130 traditional IRA contribution. Correct MAGI of $82,301 is 2.13x FPL, which is within NC's 211% + 5% disregard limit for older children."
-us,scenario_026,child3_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"The model said income was well above the children's Medicaid limit without naming a threshold. NC covers children 6-18 under Medicaid up to 211% FPL plus the 5% disregard, and household MAGI of $82,301 (2.13x FPL) falls under that limit."
-us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model named the right ~211% FPL limit but miscomputed it as about $66,000 for five people; the correct figure is about $81,600, or about $83,550 with the 5% disregard. It also tested gross wages instead of MAGI net of $2,908 in pre-tax 401(k)/IRA contributions. MAGI of $82,301 (2.13x FPL) qualifies."
-us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used the obsolete 133% FPL children's Medicaid limit and treated NC Health Choice as a separate CHIP. It then compared gross wages of $85,209 against 211% FPL, ignoring the traditional 401(k)/IRA deductions and the 5% disregard. MAGI of $82,301 (2.13x FPL) is under NC's integrated 216% effective Medicaid limit."
-us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model put income at about 260% FPL, far above the correct 2.13x. That overstatement is consistent with adding the FLSA overtime premium on top of wages that already include it, and with using an outdated FPL. Correct MAGI of $82,301 falls within NC's 211% + 5% disregard limit for older children."
-us,scenario_026,child3_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,The model measured income at about 219% FPL using the lower 2025 poverty guideline instead of the 2026 one (2.13x). It then compared that figure with a limit below NC's 211% FPL plus 5% MAGI disregard (216%) for Medicaid children ages 6-18.
-us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model treated employer-sponsored coverage as disqualifying the child from Medicaid, but ESI does not bar Medicaid eligibility. It also tested gross wages of $85,209 against 210% of an outdated FPL. MAGI net of 401(k)/IRA deductions is 2.13x FPL, within NC's 211% + 5% disregard children's Medicaid limit."
-us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model added the overtime premium on top of gross wages that already include it, inflating income to over $92,000. It then applied a 138% FPL limit to 6-18-year-olds instead of NC's post-2023 integrated limit of 211% FPL plus 5% disregard. Correct MAGI of $82,301 (2.13x FPL) qualifies."
-us,scenario_026,child3_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model's income figure of about $82k was correct, but it compared it with a 133% FPL limit. NC Medicaid covers children 6-18 up to 211% FPL plus the 5% disregard (216%), and $82,301 is 2.13x FPL."
-us,scenario_026,child3_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model compared income with a 210% FPL limit and left out the 5-point MAGI disregard. NC's effective limit for children 6-18 is 216% FPL, so household MAGI at 2.13x FPL is eligible."
-us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but tested it against 133% FPL. NC moved its CHIP children into Medicaid, so the older-child Medicaid limit is 211% FPL plus 5% disregard, which 2.13x FPL meets."
-us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but applied a children's Medicaid limit below 213% FPL. NC's integrated Medicaid limit for ages 6-18 is 211% FPL plus 5% disregard (216%), so the child qualifies."
-us,scenario_026,child3_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model tested gross wages of $85,209 instead of MAGI, which subtracts $2,778 in traditional 401(k) and $130 in traditional IRA contributions to give $82,301. It also used a 210% limit without the 5% disregard; MAGI of 2.13x FPL is within NC's 216% effective limit."
-us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit for children 6-18, which NC replaced when it merged NC Health Choice into Medicaid. Under the 211% FPL + 5% disregard limit the child qualifies; even the model's own understated MAGI of $61,093 is well under it."
-us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model said income exceeded the threshold without naming one. NC Medicaid covers children 6-18 up to 211% FPL plus 5% disregard, and household MAGI is 2.13x FPL, below that limit."
-us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's MAGI of $82,301 was correct, but it applied a 133% FPL limit and sent the child to CHIP. NC folded CHIP children into Medicaid with a 211% FPL + 5% disregard limit, and 2.13x FPL qualifies under OLDER_CHILD Medicaid."
-us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an outdated 133% FPL threshold for children's Medicaid. NC's limit for ages 6-18 is 211% FPL plus 5% disregard, so household MAGI at 2.13x FPL qualifies; the model's own understated $61,093 figure would also qualify."
-us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no income test. NC's Medicaid limit for children 6-18 is 211% FPL plus 5% disregard, and household MAGI of $82,301 (2.13x FPL) is within it, so the child is eligible."
-us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model said income exceeded NC's limit without stating one. NC's children's Medicaid limit for ages 6-18 is 211% FPL plus 5% disregard (216%), which covers the household's 2.13x FPL MAGI."
-us,scenario_026,child3_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model claimed income exceeded the limit, but NC Medicaid covers children 6-18 up to 211% FPL plus 5% disregard. Household MAGI after 401(k)/IRA deductions is 2.13x FPL, which is under that limit."
-us,scenario_026,child3_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,The model applied a children's Medicaid limit below the household's 2.13x FPL MAGI. NC's integrated Medicaid limit for ages 6-18 is 211% FPL plus 5% disregard.
-us,scenario_026,child3_medicaid_eligible,gemini-4-argon,llm_error,thresholds_rates,False,"The model assumed MAGI was above NC's limit for children. The older-child Medicaid limit is 211% FPL plus 5% disregard (216%), and household MAGI is 2.13x FPL."
-us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model computed about 213% FPL correctly but used a 133% Medicaid limit and a separate 210% CHIP limit. NC merged CHIP children into Medicaid at 211% FPL plus 5% disregard (216%), so 213% qualifies. ESI does not bar Medicaid eligibility."
-us,scenario_026,child3_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model compared gross wages of $85,209 with the 211% limit, skipping the $2,908 in traditional 401(k)/IRA deductions and the 5% MAGI disregard. MAGI of $82,301 is 2.13x FPL, within the 216% effective limit."
-us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model used a 'benchmark Medicaid threshold' below the household's income, but NC covers school-age children under Medicaid up to 211% FPL plus 5% disregard. Household MAGI is 2.13x FPL, which qualifies."
-us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model concluded the child was ineligible without any threshold test. NC's OLDER_CHILD Medicaid limit is 211% FPL plus 5% disregard, and household MAGI of 2.13x FPL is within it."
-us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model tested pre-deduction earnings of about $85,209 against a children's Medicaid limit below NC's actual 211% FPL plus 5% disregard. MAGI after traditional 401(k)/IRA contributions is $82,301 (2.13x FPL), which qualifies."
-us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's estimated NC threshold for children was too low. Medicaid covers children 6-18 up to 211% FPL plus 5% disregard, which includes household MAGI at 2.13x FPL."
-us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model applied a limit for a 9-year-old below 2.13x FPL. NC's integrated Medicaid limit for ages 6-18 is 211% FPL plus 5% disregard (216%), so the child is eligible."
-us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model placed household income above NC's child limit, but that limit is 211% FPL plus 5% disregard for ages 6-18. MAGI of $82,301 (2.13x FPL) is below it."
-us,scenario_026,child3_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"The model applied an age-specific limit below the household's 2.13x FPL. Since CHIP was integrated into Medicaid, NC's limit for 6-18-year-olds is 211% FPL plus 5% disregard, which qualifies child3."
-us,scenario_026,child3_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"The model said income exceeded the child Medicaid limit, but NC's older-child limit is 211% FPL plus 5% disregard. Household MAGI of 2.13x FPL is within it."
-us,scenario_026,child3_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model computed MAGI correctly at $82,301 but applied a limit below 213% FPL. NC Medicaid covers children 6-18 up to 211% FPL plus 5% disregard (216%), so the child is eligible."
-us,scenario_026,child3_medicaid_eligible,gpt-6.1-terra,llm_error,thresholds_rates,False,"The model treated the household as too high-income for children's Medicaid. NC's integrated children's Medicaid limit for ages 6-18 is 211% FPL plus 5% disregard, and MAGI is 2.13x FPL."
-us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model used a threshold below NC's 211% FPL + 5% disregard limit for children 6-18 and treated ESI as relevant. ESI does not bar Medicaid, and MAGI of 2.13x FPL qualifies."
-us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model applied the pre-2023 133% FPL limit for children 6-18. After NC Health Choice was merged into Medicaid, the limit is 211% FPL plus 5% disregard, which the household's 2.13x FPL MAGI satisfies."
-us,scenario_026,child3_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model used a 133% FPL limit for children 6-18 instead of NC's 211% FPL + 5% disregard Medicaid limit. Both its understated MAGI of $74,555 and the correct $82,301 (2.13x FPL) qualify under the right limit."
-us,scenario_026,child3_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,The model applied a children's limit below the household's 2.13x FPL. NC Medicaid covers ages 6-18 up to 211% FPL plus 5% disregard.
-us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,The model overstated income at about 240% FPL instead of 2.13x by ignoring the 401(k)/IRA deductions and using an outdated FPL. It also applied the obsolete 133% FPL children's limit rather than NC's 211% + 5% disregard Medicaid limit.
-us,scenario_026,child3_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model used a 133% FPL limit for ages 6-18 and treated employer coverage as disqualifying. NC's integrated Medicaid limit is 211% FPL plus 5% disregard, ESI does not bar Medicaid, and MAGI of 2.13x FPL qualifies."
-us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for child3_medicaid_eligible, so there was no answer to score."
-us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model applied a 133%/138% FPL limit for children 6-18 instead of NC's 211% FPL plus 5% disregard. It also overstated MAGI at about 226% FPL instead of 2.13x, consistent with using an older, lower poverty guideline."
-us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model treated income as well above NC's children's Medicaid limit. That limit for ages 6-18 is 211% FPL plus 5% disregard, and household MAGI is only 2.13x FPL."
-us,scenario_026,child3_medicaid_eligible,muse-spark-1.3,llm_error,thresholds_rates,False,"The model overstated MAGI at about 221% FPL instead of 2.13x by using an outdated, lower poverty guideline. It compared that figure with 211% without adding the 5% MAGI disregard, and treated ESI as relevant; the correct 2.13x is under the 216% effective limit."
-us,scenario_026,child3_medicaid_eligible,muse-wonder,llm_error,thresholds_rates,False,"The model assumed MAGI was above the NC child limit. That limit for ages 6-18 is 211% FPL plus 5% disregard, and MAGI of $82,301 (2.13x FPL) is below it."
-us,scenario_026,child3_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model placed children 6-18 under a 133% FPL limit. NC's integrated Medicaid limit is 211% FPL plus 5% disregard, so even its own estimate of about 198% FPL qualifies, as does the correct 2.13x."
-us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit with an understated five-person FPL of $33,700 and tested gross wages rather than MAGI. NC's children's Medicaid limit is 211% FPL plus 5% disregard, and MAGI of $82,301 (2.13x FPL) qualifies."
-us,scenario_026,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model called income too high without a threshold test. NC's Medicaid limit for children 6-18 is 211% FPL plus 5% disregard, which covers the household's 2.13x FPL MAGI."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It reached about $1,576 after exemptions, the 2.75% rate and the $200 retirement credit. It then made an unexplained downward 'graduated structure' adjustment to $875. That adjustment runs the wrong way: the 2026 schedule adds a $332 base amount, and the correct liability is $1,916.61."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its exemptions ($1,900 × 2) and $200 retirement credit were right, but it taxed only 2.75% of the excess over $26,050. It left out the roughly $332 base amount in Ohio's 2026 schedule. It also skipped the $180.60 unreimbursed-medical deduction, so it came in about $327 low."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It got the $332 base, the $1,900 exemptions and the $200 credit right. It missed only Ohio's unreimbursed medical deduction: $6,500 of premiums plus $800 of medical expenses above 7.5% of FAGI is $180.60, which lowers taxable income and the tax by about $5."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It wrongly subtracted a federal standard deduction (Ohio starts from FAGI and allows only personal exemptions). It applied obsolete brackets running up to 5.75% instead of the 2026 schedule of $332 plus 2.75% above $26,050, and it ignored the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It gave a rounded $1,000 with no computation. The correct path is OAGI $94,744.70, less $3,800 of exemptions, then $332 plus 2.75% above $26,050 ($2,116.61), less the $200 retirement credit, for $1,916.61. Its figure fits a large understatement of the 2.75% bracket tax."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It used $1,850 exemptions instead of $1,900, replaced the $332 base with an arbitrary 'modest amount', and claimed no nonrefundable credits apply. It missed the $200 retirement income credit for the spouse's $32,200 pension and the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It computed $1,790 of tax and said the retirement and joint filing credits apply, then reported a higher $2,426 that does not follow from its own steps. It also left out the $332 base amount and wrongly invoked the joint filing credit, which needs qualifying income from both spouses."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It used $2,500 exemptions instead of the $1,900 over-$80k tier and the 2024 base of about $360 instead of $332. It also omitted the medical deduction, and its $1,866 implies about $250 of credits rather than the single $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It explicitly assumed the 2026 schedule has no base amount and taxed only 2.75% of the excess over $26,050. Ohio's schedule adds about $332. It also skipped the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the $2,400 lowest-income exemption instead of $1,900 for OAGI above $80k, and it omitted the $332 base amount. It also wrongly concluded that Ohio has no deduction for the head's $6,500 premiums, when they belong on the unreimbursed medical worksheet with a 7.5% floor."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"It used $2,350 exemptions, left out the $332 base amount, and subtracted about $500 of credits, including a senior credit that requires age 65 (the head is 61 and the spouse 57). Only the $200 retirement income credit applies."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It applied 2.75% above $26,050 with no base amount, missing the roughly $332 that Ohio's 2026 schedule adds for income above the zero bracket. It also ignored the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used $1,000 exemptions instead of $1,900 and made up a schedule of 2.5% on the first $26,500 and 3.5% above, instead of 0% to $26,050 and then $332 plus 2.75%. It also omitted the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It deducted a single $2,400 exemption instead of two $1,900 exemptions and applied obsolete brackets (3.226%, 3.688%) instead of $332 plus 2.75% above $26,050. It also said no nonrefundable credits apply, missing the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used $2,800 exemptions instead of $1,900, omitted the $332 base amount, and claimed no nonrefundable credits, missing the $200 retirement income credit for the $32,200 pension."
+us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It made up a 3.5% bracket above $46,100 and used $2,400 exemptions instead of $1,900. It took a 5% joint filing credit even though the head has no qualifying income, and it skipped the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $21,208 of employer-sponsored premiums from FAGI even though the head has no wages to exclude them from, and it added $50 of over-the-counter costs to medical expenses. Together these inflated the medical deduction to $1,821. It also used $2,400 exemptions and omitted the $332 base amount."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave $1,695 with no derivation. That is about $220 below the correct $1,916.61, which fits a bare 2.75% calculation without most of the $332 base amount in Ohio's 2026 schedule."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $1,762 equals 2.75% of ($90,125 − $26,050), so it used $2,400 exemptions, no $332 base amount and no credits. It missed the $200 retirement income credit and the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used $2,700 exemptions instead of $1,900 and taxed only 2.75% of the excess, without the $332 base amount. It also never applied the $200 retirement income credit or the medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave $1,280.50 with no derivation. The correct steps give $90,944.70 of taxable income, $2,116.61 of tax under $332 plus 2.75% above $26,050, and $1,916.61 after the $200 retirement credit. Its figure understates the bracket tax by several hundred dollars."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its exemptions and $200 retirement credit were correct, but it applied 2.75% above $26,050 without the roughly $332 base amount in the 2026 schedule. It also ignored the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $1,694.06 equals 2.75% of ($94,925 − $26,050) minus $200. That means it applied no personal exemptions (two at $1,900 apply), no medical deduction, and no $332 base amount."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It wrongly claimed Ohio exemptions are $0 above $80,000 MAGI. They are $1,900 each in that tier. It also omitted the $332 base amount, the $180.60 medical deduction and the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"Its $2,150.25 base tax uses the 2024 base of $360.69 instead of the 2026 base of about $332. It also skipped the $180.60 medical deduction, so its result is about $34 too high even though the exemptions and $200 credit are right."
+us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It subtracted a $30,000 federal standard deduction (Ohio allows none) and $2,400 exemptions, then applied 2.75% to the whole amount, ignoring the $26,050 zero bracket and the $332 base. It also omitted the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Its reasoning used obsolete pre-2019 multi-bracket rates up to 4.396% and a joint filing credit the head cannot qualify for, reaching $3,245. It then submitted an unrelated $1,260.80. Neither number uses the 2026 schedule of $332 plus 2.75% above $26,050 or the $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It assumed low-income offsets wipe out the liability. But taxable nonbusiness income is about $90,945, which produces $2,116.61 of tax under the 2026 schedule, and only the $200 retirement credit reduces it."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It applied a 'standard deduction' that Ohio does not have, in place of the $180.60 medical deduction and the $1,900 personal exemptions. Its $1,340 understates tax under the $332 plus 2.75% schedule, which comes to $1,916.61 after the $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It added the $50 of over-the-counter costs to the medical worksheet, which raised the deduction to $230.63 instead of $180.60. It also used the 2024 base of $360.69 instead of the 2026 base of about $332."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It used $2,400 exemptions instead of $1,900 for OAGI over $80k, taxed only 2.75% of the excess without the $332 base, and claimed no nonrefundable credits. It missed the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $2,150.25 pre-credit tax uses the 2024 base of $360.69 instead of the 2026 base of about $332. It also omitted the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It wrongly excluded the $32,200 private pension from Ohio income, which Ohio taxes in full. It also used $2,150 exemptions, claimed a joint filing credit the head cannot qualify for, and cut the retirement credit to $25 instead of $200."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It put the $21,208 of employer-sponsored premiums on the unreimbursed medical worksheet and deducted $21,388.63. Only the $6,500 of premiums and $800 of medical expenses count, which gives $180.60. The inflated deduction pushed OAGI into the $2,150 exemption tier instead of $1,900."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It applied 2.75% above $26,050 and the $200 retirement credit, but left out the roughly $332 base amount in Ohio's 2026 schedule. It also left out the $180.60 medical deduction, so it came in about $327 low."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Its exemptions and $200 credit were correct, but it taxed only 2.75% of the excess over $26,050 and dropped the $332 base amount. It also ignored the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It deducted the full $6,500 of premiums as if the head had no access to employer-subsidized coverage. Because the head has ESI, the premiums go on the medical worksheet with the $800 of expenses and are deductible only above the 7.5% FAGI floor, which gives $180.60."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It applied 2.75% above $26,050 and the $200 retirement credit without the roughly $332 base amount in the 2026 schedule. Its exemption estimate also came out slightly above $3,800."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It claimed deductions and credits reduce Ohio tax to zero. In fact, $90,944.70 of taxable income carries $2,116.61 of tax under $332 plus 2.75% above $26,050, and only the $200 retirement credit offsets it."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used $2,400 exemptions instead of $1,900, omitted the $332 base amount, and stated zero nonrefundable credits. It missed the $200 retirement income credit for the spouse's pension."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"It subtracted a 5% joint filing credit even though the head has no qualifying income, and it omitted the $332 base amount. It also skipped the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its exemptions and $200 credit were correct, but it treated $26,050 as a pure zero bracket and taxed only 2.75% of the excess. It missed the roughly $332 base amount and the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used $2,650 exemptions and made up a 3.5% bracket above $46,100, when the 2026 schedule is $332 plus 2.75% up to $100,000. It also omitted the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used $2,400 exemptions instead of $1,900, applied 2.75% without the $332 base, and stated no nonrefundable credits apply. It missed the $200 retirement income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used $2,500 exemptions instead of $1,900 and taxed 2.75% of the excess without the $332 base amount. It also never applied the $200 retirement income credit or the medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Its $1,900 exemptions and $200 credit were correct, but it computed tax as 2.75% of the excess over $26,050 with no base amount, missing the roughly $332 in Ohio's 2026 schedule. It also skipped the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"It invoked senior/aged adjustments that do not apply (the head is 61 and the spouse 57) and treated taxable income as near zero. The $94,925 of wages and pension produces $2,116.61 of tax before the $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used $2,400 exemptions to reach $90,125 and applied 2.75% without the $332 base amount. It omitted the $200 retirement income credit and the $180.60 medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"Its exemptions and $200 retirement credit were correct, but it taxed only 2.75% above $26,050, leaving out the roughly $332 base amount. It also ignored the $180.60 unreimbursed medical deduction."
+us,scenario_025,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It applied no personal exemptions (two $1,900 exemptions apply at this income), no $332 base amount, and no $200 retirement income credit. It checked only the joint filing and senior credits."
+us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It gave about $2,651 without applying the exemptions, the $332 plus 2.75% schedule or the $200 retirement credit. That overstates the correct $1,916.61, which fits higher, invented bracket rates."
+us,scenario_025,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It made up a $9,300 MFJ standard deduction and taxed the first $26,050 at 2.75% and the rest at 3.5%. The 2026 schedule is 0% to $26,050, then $332 plus 2.75%. It also took a senior credit for a 61-year-old instead of the $200 retirement income credit."
+us,scenario_026,child1_chip_eligible,claude-fable-5.2,llm_error,categorical_eligibility,False,"The model said child1 was not Medicaid eligible and treated the roughly 216% FPL limit as a CHIP threshold. In North Carolina, that band is Medicaid coverage for older children, so child1 is Medicaid eligible under OLDER_CHILD, which rules out CHIP. The model checked the income limit but never applied the rule that Medicaid eligibility excludes a child from CHIP."
+us,scenario_026,child1_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated 211% FPL plus the 5% disregard as North Carolina's CHIP limit. Since NC Health Choice merged into Medicaid, that limit belongs to Medicaid for school-age children. Child1 therefore qualifies for Medicaid under OLDER_CHILD, and Medicaid eligibility rules out CHIP."
+us,scenario_026,child1_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated NC Health Choice as a separate CHIP program with a 211% FPL limit and called child1 CHIP eligible. That coverage now runs through Medicaid, so child1 qualifies for Medicaid under OLDER_CHILD and cannot also be eligible for CHIP. The model also understated household income at $61,093."
+us,scenario_026,child1_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"The model assumed North Carolina Medicaid for a school-age child stops at 133% FPL and that a separate CHIP (NC Health Choice) covers up to 211% FPL. North Carolina now covers children at this income level through Medicaid, so child1 qualifies under OLDER_CHILD. Medicaid eligibility makes the child ineligible for CHIP."
+us,scenario_026,child1_chip_eligible,grok-4.7,llm_error,categorical_eligibility,False,"The model placed child1 above the Medicaid income limit but below a separate 211% FPL CHIP limit for NC Health Choice. That program now runs through Medicaid, so child1 qualifies for Medicaid under OLDER_CHILD, which rules out CHIP."
+us,scenario_026,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for child1_chip_eligible, so there was nothing to parse. The correct derivation finds child1 Medicaid eligible under OLDER_CHILD, which makes the CHIP answer 0."
+us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"It got MAGI right (about $82,300), but compared it with a 133% + 5% (138%) FPL limit for children ages 6–18. That limit is out of date: since NC Health Choice merged into Medicaid in 2023, the older-child limit is 211% FPL (216% with the disregard). MAGI of 2.13 × FPL is within it."
+us,scenario_026,child1_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"It computed the correct income ratio (about 213% FPL), but used the old 133% FPL Medicaid limit for children ages 6–18. NC's older-child Medicaid limit is now 211% FPL plus the 5-point disregard (216%), so 213% qualifies."
+us,scenario_026,child1_medicaid_eligible,claude-fable-5.2,llm_error,thresholds_rates,False,"It correctly put income at about 213% FPL, but applied an outdated 138% FPL limit for children ages 6–18. Since 2023, NC Medicaid covers this age group up to 211% FPL (216% with the disregard), so Child 1 is eligible."
+us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"It treated employer-sponsored insurance as disqualifying, but ESI does not bar Medicaid; it only bars CHIP. It also understated the 2026 FPL for five people at $27,465 (the engine's figure is about $38,600) and tested gross wages instead of MAGI of $82,301, which is 2.13 × FPL and within NC's 216% effective older-child limit."
+us,scenario_026,child1_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"It said household income is ""well above"" the child Medicaid limit, but MAGI of $82,301 is 2.13 × FPL. That is within NC's older-child limit of 211% FPL plus the 5-point disregard. The answer matches the pre-2023 133%/138% limit for ages 6–18."
+us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a 138% FPL limit (about $51,957) for children ages 6–18. Since NC Health Choice was absorbed into Medicaid, the limit for this group is 211% FPL (216% with the disregard), and the household's 2.13 × FPL MAGI is within it."
+us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied a 138% FPL limit for NC children ages 6–18 and even put that at only about $45,000. The correct older-child limit is 211% FPL plus the 5-point disregard. The household's MAGI of $82,301 (2.13 × FPL) is within it."
+us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It identified the right 211% limit for ages 6–18 but put income at about 260% FPL. Correct MAGI is wages minus traditional 401(k) and IRA contributions, $82,301, which is 2.13 × FPL. That is within the 211% limit once the 5-point MAGI disregard (216%) is applied."
+us,scenario_026,child1_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Its 219% figure is $82,301 divided by the 2025 FPL for five ($37,650), not the 2026 FPL the engine uses. The correct ratio is 2.13 × FPL, which is within NC's 211% + 5-point disregard limit for older children."
+us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"It ruled Child 1 out because the child has employer-sponsored insurance. Medicaid has no uninsured requirement, and ESI only blocks CHIP. On income alone, 2.13 × FPL qualifies under NC's 216% effective older-child limit."
+us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It counted about $93,000 of income by adding the $7,746 FLSA overtime premium to wages that already include overtime, and skipped the 401(k) and IRA deductions. It also understated 211% FPL for five as about $67,000. Correct MAGI is $82,301, or 2.13 × FPL, which is within the 216% effective limit."
+us,scenario_026,child1_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It got income of about $82k right, but compared it with the pre-2023 133% FPL limit for children. NC's older-child Medicaid limit is now 211% FPL plus the 5-point disregard, and $82,301 (2.13 × FPL) is within it."
+us,scenario_026,child1_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It counted household income as about $93k by adding the FLSA overtime premium to wages that already include it, and ignored the pre-tax 401(k) and IRA deductions. It also left out the 5-point disregard. Correct MAGI is $82,301 (2.13 × FPL), which is under the 216% effective older-child limit."
+us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"It got MAGI right ($82,301) but tested it against the old 133% FPL limit. Since 2023, NC Medicaid covers children ages 6–18 up to 211% FPL (216% with the disregard), so 2.13 × FPL qualifies."
+us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed MAGI of $82,301 correctly but said it exceeds the child limit. That figure is 2.13 × FPL, which is within NC's older-child limit of 211% FPL plus the 5-point disregard. The model applied a limit below 213% FPL, either the pre-2023 133%/138% cap or 211% without the disregard."
+us,scenario_026,child1_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It compared gross wages of $85,209 with a 210% FPL limit. It did not subtract the $2,778 traditional 401(k) and $130 IRA contributions (MAGI $82,301) and did not apply the 5-point disregard. The correct 2.13 × FPL ratio is under the 216% effective limit."
+us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a 147% FPL limit for NC children ages 6–18 instead of 211% FPL plus the 5-point disregard. Even its own MAGI of $61,093 (which wrongly subtracts the head's $21,208 ESI premiums) is far below the correct limit. The engine's 2.13 × FPL also qualifies."
+us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It said household income is above the limit without showing figures. MAGI of $82,301 is 2.13 × FPL, within NC's 211% + 5-point disregard limit for children ages 6–18. The answer is consistent with the outdated 133%/138% limit."
+us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It had the correct MAGI ($82,301) but used a 133% FPL Medicaid cap and sent the child to CHIP instead. NC Health Choice was folded into Medicaid in 2023, so children ages 6–18 up to 211% FPL (216% with the disregard) are Medicaid-eligible. At 2.13 × FPL, Child 1 qualifies."
+us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a 133% FPL child limit instead of NC's current older-child limit of 211% FPL plus the 5-point disregard. Its $61,093 income figure, which wrongly subtracts the head's ESI premiums, would qualify under the correct limit. The engine's MAGI of 2.13 × FPL also qualifies."
+us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It said Child 1 is ineligible without giving a reason. Correct MAGI is $82,301, or 2.13 × FPL, which is within NC's older-child Medicaid limit of 211% FPL plus the 5-point disregard. The answer reflects the outdated 133%/138% limit."
+us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"It said income exceeds NC's limit, but MAGI of $82,301 (2.13 × FPL) is under the 216% effective older-child Medicaid limit, which has applied since NC Health Choice merged into Medicaid. It applied a lower, outdated limit for children ages 6–18."
+us,scenario_026,child1_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"It said household income exceeds the limit. MAGI after the 401(k) and IRA deductions is $82,301, or 2.13 × FPL, which is within NC's 211% + 5-point disregard limit for children ages 6–18. The model used a lower limit."
+us,scenario_026,child1_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"It said income exceeds NC's children's Medicaid limit, but the older-child limit is 211% FPL plus the 5-point disregard. The household's 2.13 × FPL MAGI is within it, so the model applied a lower (pre-2023) limit."
+us,scenario_026,child1_medicaid_eligible,gemini-4-argon,llm_error,thresholds_rates,False,"It said MAGI is above the NC child limit, but MAGI of $82,301 is 2.13 × FPL. That is within the 216% effective older-child Medicaid limit (211% + 5-point disregard), so the model used a limit that is too low."
+us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,It ruled Child 1 out because of employer-sponsored insurance. ESI does not affect Medicaid eligibility; it only bars CHIP. Child 1's MAGI of 2.13 × FPL meets NC's older-child Medicaid limit.
+us,scenario_026,child1_medicaid_eligible,glm-5.3,llm_error,taxable_income_or_deductions,False,"It tested gross wages of $85,209 against 211% FPL. It did not subtract the traditional 401(k) and IRA contributions (MAGI $82,301) or add the 5-point MAGI disregard. The correct 2.13 × FPL ratio is within the 216% effective limit."
+us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"It said earnings exceed the Medicaid threshold without giving figures. MAGI of $82,301 is 2.13 × FPL, within NC's older-child limit of 211% FPL plus the 5-point disregard. The model applied an outdated lower limit."
+us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"It treated the household's wage level as disqualifying without testing it against the older-child limit. MAGI of 2.13 × FPL is under NC's 216% effective Medicaid limit for children ages 6–18, so Child 1 qualifies."
+us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"It said income is above the limit for school-age children. NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard, and the household's MAGI of $82,301 (2.13 × FPL) is within it."
+us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its estimated child Medicaid threshold was too low. NC covers children ages 6–18 up to 211% FPL plus the 5-point disregard (216%), and MAGI of 2.13 × FPL is within that."
+us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"It said MAGI exceeds NC's limit for an 11-year-old, but MAGI of $82,301 is 2.13 × FPL. The older-child limit is 211% FPL plus the 5-point disregard, so Child 1 is eligible."
+us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It said income is above the NC child limit. The household's 2.13 × FPL MAGI is under the 216% effective older-child limit (211% + 5-point disregard), so the model applied a limit that is too low."
+us,scenario_026,child1_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"It said income exceeds the limit for an 11-year-old. NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard, and MAGI of 2.13 × FPL is within it."
+us,scenario_026,child1_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"It said income exceeds the child Medicaid limit, but MAGI of $82,301 is 2.13 × FPL. That is under NC's 216% effective limit for children ages 6–18."
+us,scenario_026,child1_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"It got MAGI right at $82,301 but said it exceeds the limit for five people. That MAGI is 2.13 × FPL, within NC's older-child limit of 211% FPL plus the 5-point disregard, so the model used a limit below 213% FPL."
+us,scenario_026,child1_medicaid_eligible,gpt-6.1-terra,llm_error,thresholds_rates,False,"It said the household's income rules out income-based children's Medicaid. MAGI of 2.13 × FPL is within NC's 216% effective older-child limit, which has applied since NC Health Choice was absorbed into Medicaid."
+us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,It cited both income and ESI. Employer coverage does not bar Medicaid; it only bars CHIP. MAGI of 2.13 × FPL is within NC's 211% + 5-point disregard limit for older children.
+us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"It applied a 133% FPL limit for children ages 6–18, which is out of date. Since NC Health Choice merged into Medicaid in 2023, the limit is 211% FPL plus the 5-point disregard, and MAGI of 2.13 × FPL qualifies."
+us,scenario_026,child1_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"It used a 133% FPL (about $51,000) limit for children ages 6–18 instead of NC's current 211% FPL plus the 5-point disregard. Its own MAGI of $74,555 and the correct $82,301 (2.13 × FPL) are both within the correct limit."
+us,scenario_026,child1_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"It said NC Medicaid for children ages 6–18 stops at about 138% FPL. The current limit is 211% FPL plus the 5-point disregard (216%), so MAGI of 2.13 × FPL qualifies."
+us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"It applied the outdated 133% FPL base limit and overstated income at about 240% FPL. Correct MAGI is $82,301, or 2.13 × FPL, which is within NC's 211% + 5-point disregard older-child Medicaid limit."
+us,scenario_026,child1_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It said Medicaid for ages 6–18 is limited to 133% FPL. NC's older-child Medicaid limit is 211% FPL plus the 5-point disregard, so MAGI of 2.13 × FPL qualifies, and the uninsured requirement it mentioned applies only to CHIP."
+us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no usable value or explanation for child1_medicaid_eligible, so the answer could not be scored."
+us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"It used a 133% (138% with disregard) FPL limit for children ages 6–18 and overstated income at about 226% FPL. The correct older-child limit is 211% FPL plus the 5-point disregard, and MAGI of 2.13 × FPL is within it."
+us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"It called income ""well above"" NC's children's Medicaid limit, but MAGI of $82,301 is 2.13 × FPL. That is under the 216% effective older-child limit, so the model applied the outdated lower limit."
+us,scenario_026,child1_medicaid_eligible,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It put MAGI at about 221% FPL against a 211% limit, when the correct ratio is 2.13 × FPL. It also left out the 5-point disregard, which raises the limit to 216%. Citing ESI is also wrong, since employer coverage does not bar Medicaid."
+us,scenario_026,child1_medicaid_eligible,muse-wonder,llm_error,thresholds_rates,False,"It said MAGI is above NC's child limit, but MAGI of 2.13 × FPL is within the older-child limit of 211% FPL plus the 5-point disregard."
+us,scenario_026,child1_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"It placed the children in a 6–18 Medicaid group capped at 133% FPL. That cap was replaced when NC Health Choice merged into Medicaid; the limit is now 211% FPL plus the 5-point disregard. Even its own 198% FPL estimate qualifies, as does the correct 2.13 × FPL."
+us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"It correctly found income within the children's limit, then disqualified Child 1 for having employer-sponsored insurance. ESI does not bar Medicaid eligibility; it only bars CHIP."
+us,scenario_026,child1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"It said income is too high for child Medicaid without giving a limit. MAGI of $82,301 is 2.13 × FPL, within NC's older-child limit of 211% FPL plus the 5-point disregard."
+us,scenario_026,child2_chip_eligible,claude-fable-5.2,llm_error,categorical_eligibility,False,"The model said Child 2 was ""not Medicaid eligible"" and assigned the child to CHIP because income of about 213% FPL fell under a 216% limit. In North Carolina that income band is covered by older-child Medicaid, so Child 2 is Medicaid-eligible as an OLDER_CHILD. That Medicaid eligibility rules out CHIP."
+us,scenario_026,child2_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model compared MAGI of $82,301 with a 216% FPL limit and treated that limit as a separate CHIP program. In North Carolina that limit is the older-child Medicaid standard, so Child 2 is Medicaid-eligible (OLDER_CHILD). CHIP covers only children who do not qualify for Medicaid, so Child 2 is not CHIP-eligible."
+us,scenario_026,child2_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated income below a 211% FPL threshold as qualifying for NC Health Choice CHIP. North Carolina has merged NC Health Choice into Medicaid, so a school-age child in that band is Medicaid-eligible under the OLDER_CHILD category. That Medicaid eligibility bars CHIP. The model also understated household income at $61,093."
+us,scenario_026,child2_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"The model assumed Medicaid for a child aged 6–18 ends at 133% FPL and that 133–211% FPL is a CHIP band. In North Carolina that band is part of older-child Medicaid, so Child 2 is Medicaid-eligible (OLDER_CHILD) and therefore not CHIP-eligible."
+us,scenario_026,child2_chip_eligible,grok-4.7,llm_error,categorical_eligibility,False,"The model placed Child 2 in a CHIP range ""above Medicaid and at or below 211% FPL."" North Carolina's older-child Medicaid already covers that range, so Child 2 is Medicaid-eligible under OLDER_CHILD. Medicaid eligibility excludes the child from CHIP."
+us,scenario_026,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value and no explanation for child2_chip_eligible, so nothing could be parsed. The correct answer is 0, because Child 2 qualifies for Medicaid under the OLDER_CHILD category and is therefore excluded from CHIP."
+us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"It computed MAGI correctly at $82,301 but used a 2026 five-person FPL that was too low (about $37,600-$37,900), which put the ratio at about 219% instead of 2.13. It then compared that against a bare 211% limit with no 5-point disregard. The correct older-child Medicaid limit is about 216% FPL, and 2.13 is under it."
+us,scenario_026,child2_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"It got the 213% FPL ratio right but applied the old 133% Medicaid limit for ages 6-18. Since the 2023 Health Choice merger, NC covers children ages 6-18 under Medicaid up to about 211% FPL (216% with the disregard), so Child 2 qualifies."
+us,scenario_026,child2_medicaid_eligible,claude-fable-5.2,llm_error,thresholds_rates,False,"It got the 213% FPL ratio right but used a 138% Medicaid limit for ages 6-18. That tier was superseded when NC moved its CHIP population into Medicaid. The older-child Medicaid limit is about 216% FPL, so 2.13 qualifies."
+us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"It assumed there were two earners, but only the spouse has wages. It never computed an FPL ratio. It also treated having employer-sponsored insurance as disqualifying, when Medicaid eligibility has no uninsured requirement. MAGI of $82,301 is 2.13× FPL, which is under NC's roughly 216% older-child Medicaid limit."
+us,scenario_026,child2_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"It said income was 'well above' the child Medicaid limit without computing anything. MAGI of $82,301 is 2.13× the 2026 five-person FPL, which is under NC's roughly 216% FPL Medicaid limit for children ages 6-18."
+us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the stale $36,580 five-person FPL and left out the $130 traditional IRA deduction. In the end it applied a 133% Medicaid limit for ages 6-18. The correct limit after the merger is about 216% FPL, and MAGI of $82,301 against the 2026 FPL is 2.13×, which qualifies."
+us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It used gross wages of about $85,000 without subtracting the 401(k) and IRA contributions, and it used the stale $36,580 FPL. It also treated the 211% tier as CHIP rather than Medicaid. NC Medicaid covers children ages 6-18 up to about 216% FPL, and the correct MAGI ratio is 2.13."
+us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"It overstated household income at about 260% FPL. The correct ratio is MAGI of $82,301 (wages minus traditional 401(k) and IRA contributions) divided by the 2026 five-person FPL, which is 2.13. That is under NC's roughly 216% older-child Medicaid limit."
+us,scenario_026,child2_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"Its 219% ratio comes from dividing $82,301 by a five-person FPL that was too low (the 2025 figure), rather than the 2026 guideline that yields 2.13. It also set the child Medicaid limit below 219% instead of about 216%, which 2.13 clears."
+us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"It made eligibility depend on an 'uninsured' requirement and ruled Child 2 out for having employer-sponsored insurance. Medicaid has no such bar; only CHIP screens for other coverage. On income alone, 2.13× FPL is under NC's roughly 216% older-child limit."
+us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It used the right 216% FPL limit but turned it into about $67,000, which implies a five-person FPL near $31,000. The actual limit is about $83,500. It also added overtime on top of gross wages, even though gross wages already include it, and never subtracted the 401(k) and IRA contributions. MAGI of $82,301 is under the limit."
+us,scenario_026,child2_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It applied the outdated 133% FPL Medicaid limit for school-age children. Since the 2023 Health Choice merger, NC Medicaid covers children ages 6-18 up to about 216% FPL, and $82,301 is 2.13× FPL."
+us,scenario_026,child2_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a 210% FPL limit without adding the 5-point MAGI disregard. The effective older-child Medicaid limit is about 216% FPL, and household MAGI at 2.13× FPL falls under it."
+us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"It computed MAGI correctly at $82,301 but compared it to a 133% FPL Medicaid limit ($51,205). That tier was replaced when NC folded Health Choice into Medicaid. The limit for ages 6-18 is about 216% FPL, and 2.13× clears it."
+us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It computed MAGI correctly at $82,301 but concluded without a stated threshold that this exceeds the child limit. Against the 2026 five-person FPL, that MAGI is 2.13×, under NC's roughly 216% older-child Medicaid limit."
+us,scenario_026,child2_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It tested gross wages of $85,209 without subtracting the $2,778 traditional 401(k) and $130 IRA contributions. It also used a 210% limit without the 5-point disregard. The correct comparison is MAGI of $82,301 (2.13× FPL) against about 216% FPL, which qualifies."
+us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the outdated 133% FPL Medicaid limit for children ages 6-18. It also subtracted the head's employer insurance premiums to reach a MAGI of $61,093, even though the head has no wages. Under the correct limit of about 216% FPL, the correct MAGI ratio of 2.13 qualifies."
+us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no threshold or computation. The correct derivation is MAGI of $82,301, which is 2.13× FPL and under NC's roughly 216% older-child Medicaid limit. Its 'No' fits applying the pre-2023 133% tier."
+us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It computed MAGI correctly at $82,301 but used a 133% Medicaid limit and assigned children above that level to CHIP. Since April 2023, NC covers children ages 6-18 under Medicaid up to about 216% FPL, so Child 2 is Medicaid-eligible."
+us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It reached a MAGI of $61,093 by wrongly subtracting the non-earning head's employer insurance premiums, and it applied the outdated 133% FPL child limit. The correct MAGI of $82,301 is 2.13× FPL, under NC's roughly 216% older-child Medicaid limit."
+us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It stated a conclusion with no income test. MAGI of $82,301 is 2.13× the 2026 five-person FPL, which is within NC's roughly 216% FPL Medicaid limit for children ages 6-18."
+us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,It asserted that income exceeds the limit without stating one. MAGI at 2.13× FPL is under NC's roughly 216% older-child Medicaid limit. Its 'No' fits using the superseded 133% tier.
+us,scenario_026,child2_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"It asserted that income exceeds the limit with no computation. MAGI of $82,301 is 2.13× FPL, which is within NC's roughly 216% FPL Medicaid limit for children ages 6-18 after the Health Choice merger."
+us,scenario_026,child2_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"It asserted that income exceeds the limit without stating one. NC Medicaid covers children ages 6-18 up to about 216% FPL, and this household's MAGI is 2.13× FPL."
+us,scenario_026,child2_medicaid_eligible,gemini-4-argon,llm_error,thresholds_rates,False,"It claimed MAGI was above the limit without numbers. MAGI of $82,301 is 2.13× the 2026 FPL, which is under NC's roughly 216% older-child Medicaid limit. Its 'No' fits the obsolete 133% tier."
+us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"It treated 133% FPL as the Medicaid limit and 210% as a separate CHIP limit. Since 2023, children ages 6-18 are covered by Medicaid up to about 216% FPL. Its own 213% ratio falls under that limit, and employer-sponsored insurance does not bar Medicaid."
+us,scenario_026,child2_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"It compared gross wages of $85,209 to a bare 211% limit. It should have used MAGI of $82,301 (after the 401(k) and IRA contributions), which is 2.13× FPL, against the roughly 216% limit that includes the 5-point disregard."
+us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"It called earnings above the threshold without computing anything. MAGI of $82,301 is 2.13× FPL, which is under NC's roughly 216% older-child Medicaid limit."
+us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave no FPL comparison. The correct derivation is MAGI of $82,301, which is 2.13× the 2026 five-person FPL and under NC's roughly 216% Medicaid limit for children ages 6-18."
+us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"It called earnings of about $85,209 'well above' the child limit. After pretax 401(k) and IRA contributions, MAGI is $82,301, or 2.13× FPL, which is under NC's roughly 216% older-child Medicaid limit."
+us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"It estimated a child Medicaid threshold below the household's income. NC's limit for ages 6-18 is about 216% FPL, and MAGI here is 2.13× FPL."
+us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"It asserted that MAGI exceeds the limit for an 11-year-old. MAGI of $82,301 is 2.13× FPL, under NC's roughly 216% older-child Medicaid limit that took effect when Health Choice merged into Medicaid."
+us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,It placed income above the child Medicaid limit with no computation. The household's 2.13× FPL MAGI is within NC's roughly 216% FPL Medicaid limit for children ages 6-18.
+us,scenario_026,child2_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"It asserted that income exceeds the limit for this age without stating one. NC Medicaid covers children ages 6-18 up to about 216% FPL, and MAGI here is 2.13× FPL."
+us,scenario_026,child2_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"It asserted that income exceeds the child limit without stating one. MAGI of $82,301 is 2.13× FPL, which is under NC's roughly 216% older-child Medicaid limit."
+us,scenario_026,child2_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"It computed MAGI correctly at $82,301 but put it above the limit. The limit is about 216% of the roughly $38,640 five-person 2026 FPL, or about $83,500, so MAGI at 2.13× FPL qualifies."
+us,scenario_026,child2_medicaid_eligible,gpt-6.1-terra,llm_error,thresholds_rates,False,It ruled out income-based children's Medicaid without computing anything. MAGI at 2.13× FPL is under NC's roughly 216% limit for ages 6-18.
+us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"It treated having employer-sponsored insurance as disqualifying, but Medicaid has no such rule, and it put income above the limit without computing anything. MAGI of $82,301 is 2.13× FPL, under NC's roughly 216% older-child Medicaid limit."
+us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"It applied a 133% FPL Medicaid limit for children ages 6-18. That tier was superseded when NC Health Choice moved into Medicaid. The limit is about 216% FPL, and MAGI is 2.13×."
+us,scenario_026,child2_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"It understated MAGI at $74,555, when the correct figure is $82,301, and applied the outdated 133% FPL limit for ages 6-18. NC's older-child Medicaid limit is about 216% FPL, and 2.13× qualifies."
+us,scenario_026,child2_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"It asserted that MAGI is above the limit for ages 6-18 without stating it. That limit is about 216% FPL after the 2023 merger, and household MAGI is 2.13× FPL."
+us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,It overstated income at about 240% FPL instead of 2.13 and used the obsolete 133% base limit. NC Medicaid covers children ages 6-18 up to about 216% FPL.
+us,scenario_026,child2_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It applied the outdated 133% FPL Medicaid limit for ages 6-18 and cited employer-sponsored insurance. NC Medicaid covers these children up to about 216% FPL with no coverage bar, and MAGI here is 2.13× FPL."
+us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for child2_medicaid_eligible, so nothing could be parsed."
+us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"It used a 133%/138% FPL limit for ages 6-18 and overstated the ratio at about 226%. MAGI of $82,301 is 2.13× the 2026 FPL, under NC's roughly 216% older-child Medicaid limit."
+us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,It called income 'well above' the limit without computing anything. MAGI at 2.13× FPL is under NC's roughly 216% Medicaid limit for children ages 6-18.
+us,scenario_026,child2_medicaid_eligible,muse-spark-1.3,llm_error,thresholds_rates,False,"It overstated the ratio at about 221% by using a five-person FPL that was too low, and it applied a bare 211% limit without the 5-point disregard. The correct ratio is 2.13, under the roughly 216% limit, and employer-sponsored insurance does not bar Medicaid."
+us,scenario_026,child2_medicaid_eligible,muse-wonder,llm_error,thresholds_rates,False,"It asserted that MAGI is above the limit with no numbers. MAGI of $82,301 is 2.13× FPL, within NC's roughly 216% older-child Medicaid limit."
+us,scenario_026,child2_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"It treated about 133% FPL as the Medicaid limit for ages 6-18. After the Health Choice merger, the limit is about 216% FPL, so even its own 198% estimate, and the correct 2.13 ratio, qualify."
+us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a five-person FPL that was too low ($33,700), compared gross wages rather than MAGI, and applied the outdated 133% limit. MAGI of $82,301 is 2.13× the 2026 FPL, under NC's roughly 216% older-child Medicaid limit."
+us,scenario_026,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"It called income too high without computing anything. MAGI of $82,301 is 2.13× FPL, which is under NC's roughly 216% FPL Medicaid limit for children ages 6-18."
+us,scenario_026,child3_chip_eligible,claude-fable-5.2,llm_error,categorical_eligibility,False,"The model declared the child 'not Medicaid eligible' and counted ~213% FPL as under a 216% CHIP limit. In NC the older-child Medicaid category now covers the former Health Choice income range, and PolicyEngine makes the child Medicaid-eligible as OLDER_CHILD. Medicaid eligibility excludes CHIP, so the answer is No."
+us,scenario_026,child3_chip_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated NC Health Choice as a separate CHIP band from 133% to 211% FPL and put the child there at ~174% FPL. NC moved Health Choice into Medicaid in 2023, so a 9-year-old at that income falls under the OLDER_CHILD Medicaid category, and Medicaid-eligible children are excluded from CHIP."
+us,scenario_026,child3_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model tested MAGI of $82,301 only against a 216% FPL CHIP limit and never checked whether the child first qualifies for Medicaid. PolicyEngine makes the child Medicaid-eligible under NC's OLDER_CHILD category, and that Medicaid eligibility bars CHIP."
+us,scenario_026,child3_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model called NC Health Choice a CHIP program and found the child eligible because $61,093 is under 211% FPL. Health Choice was merged into NC Medicaid, so the child is Medicaid-eligible as OLDER_CHILD, and a Medicaid-eligible child cannot be CHIP-eligible."
+us,scenario_026,child3_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model checked only the age and the income range it attributed to NC CHIP, and it never ran the Medicaid-first exclusion. PolicyEngine finds the 9-year-old Medicaid-eligible under the OLDER_CHILD category, which by itself makes CHIP eligibility False."
+us,scenario_026,child3_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"The model assumed NC's Medicaid limit for older children stops at 133% FPL and placed MAGI of $74,555 in a 133–211% CHIP band. In NC that band belongs to the OLDER_CHILD Medicaid category, so the child is Medicaid-eligible and therefore CHIP-ineligible."
+us,scenario_026,child3_chip_eligible,grok-4.7,llm_error,categorical_eligibility,False,"The model placed household MAGI 'above Medicaid and at or below 211% FPL' and called that CHIP. NC's older-child Medicaid category covers that range after Health Choice merged into Medicaid, so the child qualifies for Medicaid, which excludes CHIP."
+us,scenario_026,child3_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"The model called the child 'not Medicaid-eligible' and compared income to a 216% FPL Health Choice limit, using a 6-person FPL for what is a 5-person household. PolicyEngine makes the child Medicaid-eligible under NC's OLDER_CHILD category, and Medicaid eligibility rules out CHIP."
+us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"It used a 138% FPL Medicaid limit for ages 6–18 and assigned the 138–211% band to CHIP. That misses North Carolina's 2023 move of NC Health Choice into Medicaid, which brought that band into the Medicaid older-child category (211% + 5% disregard). Its own estimate of about 197% FPL is inside that Medicaid limit."
+us,scenario_026,child3_medicaid_eligible,claude-fable-5.1,llm_error,thresholds_rates,False,"It got the income ratio right (~213% FPL) but used the old 133% limit for ages 6–18. North Carolina's Medicaid older-child category now covers up to 211% FPL plus the 5% disregard, so 213% qualifies."
+us,scenario_026,child3_medicaid_eligible,claude-fable-5.2,llm_error,thresholds_rates,False,"It correctly found income at about 213% FPL but compared it with a 138% limit. Since NC Health Choice moved into Medicaid, the NC Medicaid limit for children aged 6–18 is 211% FPL plus the 5% disregard, which covers 213%."
+us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It turned a 207% FPL limit into $56,232, which is far below 207% of the roughly $38,700 poverty line for a family of 5 (about $80,000). It also left out the 5% disregard. The correct limit, 211% + 5% ≈ 216% FPL (about $83,500), covers the household's $82,301 MAGI."
+us,scenario_026,child3_medicaid_eligible,claude-haiku-5.5,llm_error,thresholds_rates,False,"It said income was 'well above' the child limit, which means it applied a limit far below 213% FPL, such as the old 133% federal minimum. NC's Medicaid older-child category covers up to 211% FPL plus the 5% disregard, so the household's 2.13× FPL MAGI qualifies."
+us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"It named the right 211% limit but put it at about $66,000, when 211% of the 2026 FPL for a family of 5 is about $81,600. With the 5% disregard the limit is about $83,500. It also compared gross wages ($85,209) instead of MAGI ($82,301), which is 2.13× FPL and inside the limit."
+us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a 133% Medicaid limit and treated 211% as a separate CHIP ceiling, even though NC Health Choice is now part of Medicaid. It then compared gross wages of $85,209, not MAGI of $82,301, with a 211% cutoff and left out the 5% disregard. At 2.13× FPL the child is under the 216% Medicaid limit."
+us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It put household income at about 260% FPL, which means it overstated income or understated the poverty line. MAGI is $85,209 − $2,778 − $130 = $82,301, or 2.13× the 2026 FPL for 5, which is under NC's 211% + 5% disregard limit for older children."
+us,scenario_026,child3_medicaid_eligible,claude-opus-5.5,llm_error,thresholds_rates,False,"It put MAGI at about 219% FPL, which matches dividing by the older 2025 poverty line instead of the 2026 one. With the 2026 FPL the ratio is 2.13, under NC's older-child Medicaid limit of 211% FPL plus the 5% disregard."
+us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It treated employer coverage as a bar to Medicaid, but having ESI does not disqualify a child from Medicaid. It also used a low FPL of $36,580 and gross wages instead of MAGI ($82,301), and left out the 5% disregard. At 2.13× FPL the child is under NC's 211% + 5% Medicaid limit."
+us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It used the old 138% limit for children aged 6–18 instead of NC's post-2023 Medicaid limit of 211% plus the 5% disregard. It also counted the overtime premium on top of wages that already include it, getting more than $92,000. Actual MAGI is $82,301, or 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Its income figure was right (about $82k) but it used a 133% FPL limit. Since NC Health Choice moved into Medicaid, NC's Medicaid older-child category covers up to 211% FPL plus the 5% disregard, which includes 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a 210% FPL limit with no 5% MAGI disregard and judged that income was above it. The effective NC older-child limit is 211% + 5% ≈ 216% FPL, and the household's MAGI is 2.13× FPL, so the child qualifies."
+us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"It computed MAGI correctly as $82,301 but tested it against 133% FPL. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, which the household's 2.13× FPL MAGI meets."
+us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It had the right MAGI of $82,301 but compared it with a child limit below 213% FPL. NC's Medicaid older-child category covers ages 6–18 up to 211% FPL plus the 5% disregard, so $82,301 (2.13× FPL) qualifies."
+us,scenario_026,child3_medicaid_eligible,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It compared gross wages of $85,209 instead of MAGI of $82,301 with a 210% limit and left out the 5% disregard. MAGI is 2.13× FPL, under NC's effective 216% older-child Medicaid limit."
+us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used the old 133% FPL limit for ages 6–18 instead of NC's 211% + 5% disregard Medicaid limit. It also wrongly subtracted the head's $21,208 ESI premiums to get a MAGI of $61,093. Actual MAGI is $82,301 (2.13× FPL), which still qualifies."
+us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It said income was above the child Medicaid limit, which means it used a limit below 213% FPL. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, and the household's MAGI is 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly computed MAGI as $82,301 but applied a 133% Medicaid limit and sent the child to CHIP. NC folded its CHIP band into Medicaid in 2023, so the older-child Medicaid limit is 211% + 5% disregard, which covers 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used the 133% FPL limit instead of NC's 211% + 5% disregard Medicaid limit for older children. It also subtracted ESI premiums to get $61,093. Even the correct MAGI of $82,301 (2.13× FPL) qualifies under the right limit."
+us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It answered ineligible with no supporting calculation, which means it used a child income limit below 213% FPL. NC's Medicaid older-child category covers up to 211% FPL plus the 5% disregard, and the household's MAGI is 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"It said income exceeded NC's limit, which means it used the old 133% child limit or left out the 5% disregard. NC's Medicaid limit for ages 6–18 is 211% FPL plus 5%, and household MAGI of $82,301 is 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"It said income exceeded the limit, which means it used a limit below 213% FPL. NC's Medicaid older-child limit has been 211% FPL plus the 5% disregard since NC Health Choice moved into Medicaid, and that covers this household's 2.13× FPL MAGI."
+us,scenario_026,child3_medicaid_eligible,gemini-3.8-flash,llm_error,thresholds_rates,False,"It said income exceeded NC's child limit, which means it used a limit below 213% FPL. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, and household MAGI is 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,gemini-4-argon,llm_error,thresholds_rates,False,"It said MAGI was above NC's child limit, which means it used a limit below 213% FPL. The NC Medicaid limit for ages 6–18 is 211% FPL plus the 5% disregard, so a MAGI of 2.13× FPL qualifies."
+us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"It correctly found income at about 213% FPL but used 133% as the Medicaid limit and 210% as a separate CHIP limit. That misses NC Health Choice's merger into Medicaid and the 5% disregard, which put the Medicaid older-child limit at about 216%. Having ESI does not disqualify the child."
+us,scenario_026,child3_medicaid_eligible,glm-5.3,llm_error,thresholds_rates,False,"It named the right 211% limit but compared gross wages ($85,209) instead of MAGI ($82,301) and left out the 5% disregard. MAGI is 2.13× FPL, under the effective 216% limit."
+us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"It said earnings were above the Medicaid limit, which means it used a child limit below 213% FPL. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, and household MAGI is 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,"It ruled out eligibility based on the wage level without testing it against a limit. NC's Medicaid older-child category covers up to 211% FPL plus the 5% disregard, and this household's MAGI of $82,301 is 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"It called $85,209 'well above' the NC child limit, which means it used a limit near the old 133% threshold. NC's Medicaid limit for ages 6–18 is 211% FPL plus the 5% disregard, which covers MAGI of $82,301 (2.13× FPL)."
+us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"It estimated NC's child Medicaid limit below 213% FPL. The actual limit for ages 6–18 is 211% FPL plus the 5% disregard, which household MAGI of 2.13× FPL meets."
+us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"It said MAGI exceeded NC's limit for a 9-year-old, which means it used a limit below 213% FPL. The NC Medicaid older-child category covers up to 211% FPL plus the 5% disregard."
+us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It said income was above the NC child Medicaid limit, which means it used a limit below 213% FPL. NC's limit for ages 6–18 is 211% FPL plus the 5% disregard, and household MAGI is 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,gpt-6-luna,llm_error,thresholds_rates,False,"It said income exceeded the age-specific NC limit, which means it used the old 133% limit for ages 6–18 or left out the disregard. NC's Medicaid older-child limit is 211% FPL plus 5%, which covers 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,gpt-6-sol,llm_error,thresholds_rates,False,"It said income exceeded the child Medicaid limit, which means it used a limit below 213% FPL. NC's older-child Medicaid limit is 211% FPL plus the 5% disregard, and household MAGI is 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,gpt-6.1-sol,llm_error,thresholds_rates,False,"It had the right MAGI of $82,301 but used an NC child limit below that amount. For a family of 5, NC's Medicaid older-child limit (211% FPL + 5% disregard ≈ $83,500) is above $82,301 (2.13× FPL)."
+us,scenario_026,child3_medicaid_eligible,gpt-6.1-terra,llm_error,thresholds_rates,False,"It ruled out income-based children's Medicaid, which means it used a limit below 213% FPL. NC's Medicaid older-child category covers up to 211% FPL plus the 5% disregard, and the household's MAGI is 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"It used a child limit below 213% FPL and counted the child's employer coverage against eligibility. Having ESI does not disqualify a child from Medicaid, and NC's older-child limit of 211% FPL plus the 5% disregard covers household MAGI of 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"It used the old 133% FPL limit for ages 6–18. Since NC Health Choice moved into Medicaid, NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, which covers MAGI of 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"It used a 133% FPL limit for ages 6–18 instead of NC's 211% + 5% disregard Medicaid limit. It also wrongly subtracted the $7,746 overtime premium from MAGI, but the overtime deduction comes after AGI and does not lower MAGI. The correct MAGI of $82,301 (2.13× FPL) qualifies."
+us,scenario_026,child3_medicaid_eligible,grok-4.7,llm_error,thresholds_rates,False,"It said MAGI was above NC's limit for ages 6–18, which means it used the old 133% threshold. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, and household MAGI is 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"It used a 133% FPL limit and overstated income at about 240% FPL. Actual MAGI is $82,301, or 2.13× the 2026 FPL for 5, under NC's older-child Medicaid limit of 211% plus the 5% disregard."
+us,scenario_026,child3_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used the old 133% FPL limit for ages 6–18 and counted employer coverage against eligibility. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, and having ESI does not disqualify a child."
+us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for child3_medicaid_eligible, so no answer could be parsed."
+us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"It used a 133%/138% limit for ages 6–18 and overstated income at 226% FPL. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, and the household's MAGI is 2.13× the 2026 FPL."
+us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"It said income was 'well above' NC's child limits, which means it used the old 133% limit for ages 6–18. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, which covers MAGI of 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,muse-spark-1.3,llm_error,thresholds_rates,False,"It overstated MAGI at 221% FPL (the correct figure is 2.13× the 2026 FPL) and left out the 5% disregard on the 211% limit. It also counted ESI against eligibility, though ESI does not disqualify a child from Medicaid."
+us,scenario_026,child3_medicaid_eligible,muse-wonder,llm_error,thresholds_rates,False,"It said MAGI was above NC's child Medicaid limit, which means it used a limit below 213% FPL. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, and household MAGI is 2.13× FPL."
+us,scenario_026,child3_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,"It used a 133% FPL limit for ages 6–18. By its own estimate income was about 198% FPL (the correct figure is 2.13×), and either figure is within NC's Medicaid older-child limit of 211% FPL plus the 5% disregard."
+us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the old 133% limit for ages 6–18, a low family-of-5 FPL of $33,700, and gross wages instead of MAGI. MAGI is $82,301, or 2.13× the 2026 FPL, under NC's 211% + 5% disregard Medicaid limit."
+us,scenario_026,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"It called income too high, which means it used a child limit below 213% FPL. NC's Medicaid older-child limit is 211% FPL plus the 5% disregard, and household MAGI is 2.13× FPL."
us,scenario_026,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It capped the nonrefundable child tax credit at $1,500 for three children; the 2026 CTC is $2,200 per child ($6,600 total) and its nonrefundable portion is applied against the full pre-credit liability before any refundable ACTC is computed, so the $4,586.51 of tax is entirely absorbed. It also used a $29,500 MFJ standard deduction instead of $32,200 and never took the $7,746.26 deduction for the qualified FLSA overtime premium. Its own stated arithmetic ($6,313 less $1,500 = $4,813) does not equal the $5,813 it submitted."
us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its derivation reached the correct conclusion — the nonrefundable CTC absorbs the entire tentative tax, 'Result near $0' — and it then submitted $4,107, a figure no step in its own work produces. It also omitted the $7,746.26 qualified-overtime deduction that brings taxable income down to $42,354.26 and tentative tax to $4,586.51."
us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It invented a ~$605 ceiling on the nonrefundable CTC by treating the $1,700-per-child refundable ACTC as consumed first; the nonrefundable credit is applied against the whole liability first and the refundable portion is the residual, so $6,600 of credit for three children eliminates the entire tax. It also missed the $7,746.26 qualified-overtime deduction, which lowers taxable income from its stated $50,001 to $42,354.26."
@@ -2365,63 +2397,65 @@ us,scenario_026,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,
us,scenario_026,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium on top of the $85,209 gross wages that already include it, then applied pre-TCJA parameters — a $16,200 standard deduction, $26,000 of personal exemptions, a 15% bracket, and a $1,000-per-child CTC. For 2026 the premium is instead deducted, the standard deduction is $32,200 with no exemptions, and the $6,600 credit for three children offsets the full $4,586.51 tentative tax."
us,scenario_026,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so no substantive computation reached the grader. The correct derivation yields $42,354.26 of taxable income after the $32,200 standard deduction and $7,746.26 overtime deduction, $4,586.51 of tentative tax, and $0 after the $6,600 nonrefundable child tax credit."
us,scenario_026,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used a fabricated $3,700 MFJ standard deduction in place of $32,200 and a fabricated $3,700 total child tax credit in place of $6,600, and it never subtracted the $2,778 traditional 401(k), the $130 IRA deduction, or the $7,746.26 overtime deduction. Its explanation states the credit fully offsets the tax while it submitted $5,931.15, contradicting itself."
-us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model left out the $7,746 OBBBA qualified-overtime deduction, so taxable income came out around $50k instead of $42,355. It also contradicted its own figures: it estimated tax at about $5,500 and then said the $6,600 CTC was fully used nonrefundably, when its own numbers left about $1,100 as ACTC. The correct tax is about $4,586.51, which leaves $2,013.49 refundable."
-us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model assigned a flat $1,000 per child as refundable without computing tax liability. The ACTC is the part of the $6,600 CTC that pre-credit tax doesn't absorb. Pre-credit tax is about $4,586.51 (after the standard and overtime deductions), so the ACTC is $2,013.49, not $3,000."
-us,scenario_026,federal_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model said the nonrefundable CTC absorbs the whole credit but never computed tax liability. After the $32,200 standard deduction and the $7,746 qualified-overtime deduction, pre-credit tax is only about $4,586.51. That is below the $6,600 CTC, so $2,013.49 is refundable."
-us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the old $2,000-per-child CTC ($6,000) instead of OBBBA's $2,200 ($6,600) and the 2025 standard deduction of $31,500. It also skipped the $7,746 overtime deduction. Its own numbers were inconsistent too: about $5,600 of tax against a $6,000 credit leaves $400 refundable, not zero."
-us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model said tax liability fully absorbed a $6,000 CTC, which uses the wrong $2,000-per-child amount instead of $2,200. It also left out the $7,746 qualified-overtime deduction, which brings pre-credit tax down to about $4,586.51 and leaves $2,013.49 of the $6,600 CTC refundable."
-us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used the correct $6,600 CTC but did not subtract the $7,746 OBBBA qualified-overtime deduction from taxable income, so it put pre-credit tax at about $5,450. The correct figure is about $4,586.51, which cut its ACTC by roughly $930."
-us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the $2,000-per-child CTC ($6,000) instead of $2,200 ($6,600) and a $30,000 standard deduction instead of $32,200. It also left out the $130 IRA deduction and the $7,746 qualified-overtime deduction. That inflated pre-credit tax to $5,815 (correct: about $4,586.51) and cut ACTC to $185."
-us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model added the $7,746 overtime premium on top of $85,209 in wages, even though it is already included, and reached an AGI of about $89k. It then never deducted that overtime under the OBBBA qualified-overtime deduction, and it used a $6,000 CTC. In fact, taxable income is $42,355 and tax is about $4,586.51, which leaves $2,013.49 of the $6,600 CTC refundable."
-us,scenario_026,federal_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model's pre-credit tax of $4,541.24 is exactly the correct liability minus 12% of $378, which means it deducted the household's $378 auto-loan interest as OBBBA qualified vehicle-loan interest. That deduction requires a new, US-assembled vehicle, and those facts are unlisted and therefore false. So tax is about $4,586.51 and the ACTC is $2,013.49."
-us,scenario_026,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model used the $2,000-per-child CTC ($6,000) instead of $2,200 and said tax absorbed all of it. It skipped the $7,746 qualified-overtime deduction, which brings pre-credit tax down to about $4,586.51 and leaves $2,013.49 of the $6,600 CTC refundable."
-us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model got earned income/AGI right at $82,301 but said the nonrefundable CTC fully offset tax. It never applied the $7,746 qualified-overtime deduction or the $2,200-per-child CTC. Pre-credit tax is about $4,586.51 against a $6,600 credit, which leaves $2,013.49 as ACTC."
-us,scenario_026,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model assumed the TCJA child credit expired, giving a $3,000 CTC ($1,000 per child). OBBBA made a $2,200-per-child CTC ($6,600) permanent, and that credit exceeds the roughly $4,586.51 pre-credit tax by $2,013.49, which is refundable."
-us,scenario_026,federal_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model's $5,516.12 tax is the correct liability before the $7,746 OBBBA qualified-overtime deduction, so it left that deduction out. It also used a $6,000 CTC instead of $6,600. With both fixed, tax is about $4,586.51 and the ACTC is $2,013.49."
-us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed 2017 rules come back in 2026, with a $1,000-per-child CTC and a $3,000 refundability threshold. OBBBA permanently set the CTC at $2,200 per child and the refundable cap at $1,700 per child. So the $6,600 credit minus about $4,586.51 of tax gives $2,013.49 of ACTC."
-us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model said no refundable credits apply without comparing the $6,600 CTC with pre-credit tax. After the standard and overtime deductions, that tax is only about $4,586.51, so $2,013.49 of the CTC is refundable ACTC."
-us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $3,000 CTC on the assumption that the TCJA provision sunset. OBBBA made the $2,200-per-child CTC permanent, so the $6,600 credit exceeds the roughly $4,586.51 pre-credit tax and $2,013.49 is refundable."
-us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's AGI of $61,093 comes from subtracting the head's $21,208 employer-sponsored insurance premiums from the spouse's wages. Those premiums don't reduce the given wage total, so AGI is $82,301, above the 3-child EITC phase-out, and EITC is $0. Its ACTC of $926 is also too low, because the correct ACTC is $2,013.49."
-us,scenario_026,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave zero without working out tax liability. Pre-credit tax is about $4,586.51 after the $32,200 standard deduction and the $7,746 qualified-overtime deduction, so $2,013.49 of the $6,600 CTC goes unused and is refunded as ACTC."
-us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model gave zero with no derivation. The $6,600 CTC exceeds the roughly $4,586.51 pre-credit tax (after the standard and overtime deductions), so the $2,013.49 excess is refundable ACTC."
-us,scenario_026,federal_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model said the nonrefundable CTC covers tax and stopped there. It left out the $7,746 qualified-overtime deduction, which brings tax down to about $4,586.51 against a $6,600 CTC, so the ACTC is $2,013.49."
-us,scenario_026,federal_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"The model said pre-credit tax fully absorbs the CTC. With the $32,200 standard deduction and the $7,746 OBBBA overtime deduction, tax is only about $4,586.51, so $2,013.49 of the $6,600 credit is refundable."
-us,scenario_026,federal_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"The model applied a $22.01 Saver's Credit ahead of the CTC, but AGI of $82,301 is above the 2026 MFJ Saver's Credit ceiling, so that credit is zero. Its base tax was also about $5 below the correct $4,586.51. Together these inflated the ACTC to $2,040.77 instead of $2,013.49."
-us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used the $2,000-per-child CTC ($6,000) instead of $2,200 and put tax at $5,676.12 without the $7,746 qualified-overtime deduction. The correct pre-credit tax is about $4,586.51, which leaves $2,013.49 of the $6,600 CTC refundable."
-us,scenario_026,federal_refundable_credits,glm-5.3,llm_error,other,False,"The model treated the $1,700 × 3 = $5,100 refundable cap as the ACTC itself. The ACTC is limited to the part of the CTC that tax doesn't absorb: $6,600 minus about $4,586.51 of tax, or $2,013.49."
-us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model only ruled out EITC and never compared the $6,600 CTC with pre-credit tax. After the standard and overtime deductions, that tax is about $4,586.51, so $2,013.49 is refundable ACTC."
-us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed there were no refundable credits without computing the CTC ordering. A $6,600 CTC against about $4,586.51 of pre-credit tax leaves $2,013.49 as refundable ACTC."
-us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used post-sunset rules with a nonrefundable $1,000-per-child credit. OBBBA permanently set the CTC at $2,200 per child with refundable ACTC up to $1,700 per child, so $2,013.49 of the $6,600 credit is refunded."
-us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $2,058.76 implies pre-credit tax of $4,541.24, which is the correct liability minus 12% of the $378 auto-loan interest. It applied the OBBBA vehicle-loan-interest deduction without the required new, US-assembled vehicle facts. Without that deduction, tax is about $4,586.51 and the ACTC is $2,013.49."
-us,scenario_026,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model's tax of about $5,532 is the liability before the $7,746 OBBBA qualified-overtime deduction. With that deduction, tax is about $4,586.51, and the unused part of the $6,600 CTC is $2,013.49, not $1,068."
-us,scenario_026,federal_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model's tax of $4,541.24 is the correct liability minus 12% of the $378 auto-loan interest, so it took the OBBBA vehicle-loan-interest deduction even though the unlisted new, US-assembled vehicle conditions are false. Tax is about $4,586.51, which leaves $2,013.49 as ACTC."
-us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model said there was no refundable CTC without computing tax liability. With the standard and $7,746 overtime deductions, tax is about $4,586.51, below the $6,600 CTC, so $2,013.49 is refundable."
-us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a $3,000 CTC on the assumption that the TCJA provision expired. OBBBA set the CTC at $2,200 per child ($6,600), which exceeds the roughly $4,586.51 pre-credit tax by $2,013.49 of refundable ACTC."
-us,scenario_026,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used the correct $6,600 CTC but put pre-credit tax at $4,671 instead of about $4,586.51, overstating taxable income by roughly $700. That understated the ACTC by about $84."
-us,scenario_026,federal_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"The model applied a $200 Saver's Credit before the CTC, but AGI of $82,301 is above the 2026 MFJ Saver's Credit income ceiling, so that credit is zero. The full roughly $4,586.51 tax should be absorbed by the CTC, which leaves $2,013.49, not $2,205."
-us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a sunset-law $3,000 CTC and overstated earned income at $90,047 by adding overtime on top of wages that already include it. It also skipped the $7,746 qualified-overtime deduction. Tax is about $4,586.51 against a $6,600 CTC, which gives $2,013.49 of ACTC."
-us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_refundable_credits, so there was no answer to score against the $2,013.49 ACTC."
-us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the correct $6,600 CTC but put pre-credit tax at $5,873 without the $7,746 OBBBA qualified-overtime deduction. The correct tax is about $4,586.51, which leaves $2,013.49 refundable, not $727."
-us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model ruled out EITC and assumed no ACTC without computing tax. Pre-credit tax is about $4,586.51 after the standard and overtime deductions, so $2,013.49 of the $6,600 CTC is refundable."
-us,scenario_026,federal_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"The model's $5,535 of nonrefundable CTC use reflects tax computed without the $7,746 OBBBA qualified-overtime deduction. With that deduction, tax is about $4,586.51 and the ACTC is $2,013.49."
-us,scenario_026,federal_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"The model's structure was right ($6,600 CTC less pre-credit tax), but it rounded pre-credit tax to $4,581 instead of $4,586.51, which understates taxable income by about $46. That produced $2,019 instead of $2,013.49."
-us,scenario_026,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model's pre-credit tax of $5,586.60 is $1,000 above the correct roughly $4,586.51 because taxable income excludes neither the full $7,746 qualified-overtime deduction nor the correct base. That shrank the unused CTC to $1,013.40 instead of $2,013.49."
-us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used the $2,000-per-child CTC ($6,000) instead of OBBBA's $2,200, and its $5,479 tax leaves out the $7,746 qualified-overtime deduction. The correct figures are a $6,600 CTC and about $4,586.51 of tax, giving $2,013.49 of ACTC."
-us,scenario_026,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"The model gave an unexplained $4,300 ACTC without limiting it to the part of the CTC that tax doesn't absorb. The ACTC is $6,600 minus about $4,586.51 of pre-credit tax, or $2,013.49."
+us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used the correct $6,600 CTC but left out the $7,746 qualified overtime deduction, so it put tax at about $5,500 instead of $4,586.51. It then said that tax used up the whole credit, but $6,600 minus $5,500 still leaves about $1,100 refundable. The correct leftover is $2,013.49."
+us,scenario_026,federal_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"Its pre-credit tax of $4,541.24 is $45.36 below the engine's $4,586.51, which equals 12% of the $378 auto-loan interest. It deducted that interest as OBBBA vehicle-loan interest, but no qualifying vehicle was established, so PolicyEngine allows no such deduction. The extra deduction pushed the refundable CTC up to $2,058.76."
+us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It assigned a flat $1,000 of refundable credit per child and never computed tax liability. The refundable CTC is the $6,600 credit minus the $4,586.51 used against tax, which is $2,013.49, not a fixed amount per child."
+us,scenario_026,federal_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It said the nonrefundable CTC soaks up all the credit. After the $32,200 standard deduction and the $7,746 qualified overtime deduction, tax is only $4,586.51, so $2,013.49 of the $6,600 CTC is left over and refundable."
+us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000 per child ($6,000) instead of $2,200 ($6,600) and left out the $7,746 overtime deduction. Even with its own numbers, $6,000 of credit against about $5,600 of tax leaves about $400 refundable, but it reported zero."
+us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used $2,000 per child ($6,000) instead of the 2026 OBBBA $2,200. It also claimed tax exceeds the credit, but with the $32,200 standard deduction and the $7,746 overtime deduction, tax is only $4,586.51. That leaves $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It estimated pre-credit tax at about $5,450 because it left out the OBBBA qualified overtime deduction for the $7,746 FLSA overtime premium. With that deduction, tax is $4,586.51 and the refundable leftover is $2,013.49, not $1,150."
+us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000 per child ($6,000) and a $30,000 standard deduction (the 2026 figure is $32,200), and it left out the $7,746 overtime deduction. That inflated tax to $5,815 and left only $185 of refundable CTC."
+us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It added the $7,746 overtime premium on top of $85,209 of wages that already include overtime, giving AGI near $89,000. It should instead have deducted the premium under OBBBA's qualified overtime deduction. It also used a $6,000 CTC, so it wrongly concluded tax used up the whole credit."
+us,scenario_026,federal_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its pre-credit tax of $4,541.24 is $45.36 low, which is 12% of the $378 auto-loan interest it wrongly deducted as OBBBA vehicle-loan interest. PolicyEngine allows no such deduction for this household. The correct tax is $4,586.51, giving $2,013.49 of refundable CTC."
+us,scenario_026,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used the pre-OBBBA $6,000 CTC and said tax used all of it. Under 2026 law the credit is $6,600, and tax after the standard and overtime deductions is only $4,586.51, so $2,013.49 is refundable."
+us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its AGI of $82,301 was right, but it did not apply the OBBBA $7,746 qualified overtime deduction, so it concluded the nonrefundable CTC covered everything. The correct tax is $4,586.51, which uses only part of the $6,600 CTC and leaves $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA would expire, giving a $1,000 per child ($3,000) CTC. OBBBA set the credit at $2,200 per child ($6,600) with up to $1,700 per child refundable, and $6,600 exceeds the $4,586.51 tax by $2,013.49."
+us,scenario_026,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $6,000 CTC instead of $6,600. It also computed tax as $5,516.12, which is exactly the result without the $7,746 qualified overtime deduction; with the deduction, tax is $4,586.51. Those two errors cut the refundable CTC from $2,013.49 to $483.88."
+us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the 2017 rules return in 2026: a $1,000 per child ($3,000) CTC with a $3,000 earnings threshold. OBBBA made the credit $2,200 per child ($6,600) with $1,700 per child refundable, so the leftover after $4,586.51 of tax is $2,013.49, not $34.15."
+us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no calculation and just said no refundable credits apply at this income. The ACTC has no income bar at $82,301 AGI: tax is $4,586.51 against a $6,600 CTC, so $2,013.49 is refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used a $3,000 CTC on the assumption that the TCJA expires. Under OBBBA the 2026 CTC is $6,600, which exceeds the $4,586.51 pre-credit tax and leaves $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $21,208 of employer-sponsored insurance premiums from AGI, getting $61,093 instead of $82,301. That made it award $930 of EITC, but earned income of $85,209 is above the 3-child phase-out, so EITC is zero. Its ACTC figure of $926 also misses the correct $6,600 − $4,586.51 = $2,013.49."
+us,scenario_026,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no calculation and reported zero, as if tax used up the whole CTC. Tax after the $32,200 standard deduction and the $7,746 overtime deduction is $4,586.51, which leaves $2,013.49 of the $6,600 CTC refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,other,False,"It gave no calculation and reported zero, which skips the ACTC altogether. With $4,586.51 of pre-credit tax against a $6,600 CTC, $2,013.49 is refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It assumed the nonrefundable CTC covers all the tax with nothing left over. With the qualified overtime deduction for the $7,746 premium and the $32,200 standard deduction, tax is only $4,586.51, so $2,013.49 of the $6,600 CTC is refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It assumed tax fully absorbs the CTC. It did not apply the $7,746 qualified overtime deduction, which brings tax down to $4,586.51 against the $6,600 credit, so $2,013.49 is refundable."
+us,scenario_026,federal_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"It applied a $22.01 Saver's Credit, but AGI of $82,301 is above the 2026 MFJ limit, so that credit is zero. It also put pre-credit tax at $4,581.24 instead of $4,586.51. Together these raised the refundable CTC to $2,040.77."
+us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used the pre-OBBBA $6,000 CTC instead of $6,600. It also put tax at $5,676.12 because it left out the $7,746 overtime deduction (correct tax is $4,586.51). Together these shrank the refundable CTC to $323.88."
+us,scenario_026,federal_refundable_credits,glm-5.3,llm_error,other,False,"It treated the $1,700 per child refundable cap as the refundable amount itself. The refundable CTC is only the part of the $6,600 credit left after offsetting the $4,586.51 tax, which is $2,013.49, not $5,100."
+us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It said refundable credits don't apply because of income and the children's ages, but the ACTC is available for children under 17 with no income bar here. The $6,600 CTC minus $4,586.51 of tax leaves $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It assumed no refundable credits because of income level and never computed the ACTC. Tax after the standard and overtime deductions is $4,586.51, so $2,013.49 of the $6,600 CTC is refundable."
+us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied the expired-TCJA $1,000 per child credit and treated it as nonrefundable. OBBBA's 2026 CTC is $2,200 per child with up to $1,700 per child refundable, and $6,600 minus $4,586.51 of tax gives $2,013.49."
+us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its answer of $2,058.76 implies pre-credit tax of $4,541.24. That is $45.36 below the engine's $4,586.51, which is 12% of the $378 auto-loan interest, so it wrongly deducted that interest even though no qualified vehicle-loan deduction is available."
+us,scenario_026,federal_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It put regular tax at about $5,532 because it left out the OBBBA qualified overtime deduction for the $7,746 FLSA overtime premium. The correct tax is $4,586.51, leaving $2,013.49 rather than $1,068 refundable."
+us,scenario_026,federal_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its pre-credit tax of $4,541.24 is $45.36 low, which is 12% of the $378 auto-loan interest it wrongly deducted as OBBBA vehicle-loan interest. The correct tax is $4,586.51 and the refundable CTC is $2,013.49."
+us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,other,False,"It said the household is too high-income for the refundable CTC, but the ACTC is not income-capped at $82,301 AGI. The $6,600 CTC is more than the $4,586.51 tax, so $2,013.49 is refundable."
+us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used a $3,000 CTC on the assumption that the TCJA expires. OBBBA's 2026 credit is $6,600, which exceeds the $4,586.51 pre-credit tax and leaves $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It put nonrefundable CTC use at $4,671, about $84 above the correct $4,586.51 tax. That gap equals 12% of $700, which is consistent with using the 2025 $31,500 standard deduction instead of the 2026 $32,200. The result understated the refundable CTC at $1,929."
+us,scenario_026,federal_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"It claimed a $200 Saver's Credit, but AGI of $82,301 is above the 2026 MFJ Saver's Credit limit, so that credit is zero. All $4,586.51 of tax should be offset by the CTC, leaving $2,013.49 refundable rather than $2,205."
+us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $3,000 CTC (assumed TCJA expiration) and inflated earned income to $90,047. That led it to put tax at $5,937 with no overtime deduction. The correct figures are a $6,600 CTC and $4,586.51 of tax, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no usable value or explanation for federal_refundable_credits, so the answer is missing."
+us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It put nonrefundable CTC use at $5,873 because it left out the $7,746 qualified overtime deduction and overstated taxable income. The correct pre-credit tax is $4,586.51, leaving $2,013.49 refundable rather than $727."
+us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,other,False,"It set the additional CTC to zero without computing tax liability. The $6,600 CTC is more than the $4,586.51 pre-credit tax, so $2,013.49 is refundable."
+us,scenario_026,federal_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It put nonrefundable CTC use at $5,535, which matches tax figured with 2025 brackets and no qualified overtime deduction for the $7,746 FLSA premium. The correct tax is $4,586.51, leaving $2,013.49 rather than $1,065."
+us,scenario_026,federal_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It put pre-credit tax at about $4,581 instead of the exact $4,586.51 and rounded the result to $2,019. Exact figures are taxable income of $42,355 after the $32,200 standard deduction and $7,746 overtime deduction, and 2026 MFJ brackets, which leave a refundable CTC of $2,013.49."
+us,scenario_026,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It put pre-credit tax at $5,586.60, about $1,000 above the correct $4,586.51, because it did not apply the OBBBA qualified overtime deduction for the $7,746 FLSA overtime premium. The refundable CTC therefore came out at $1,013.40 instead of $2,013.49."
+us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000 per child ($6,000) instead of $2,200 and put tax at $5,479 because it left out the $7,746 overtime deduction. The correct figures are $6,600 minus $4,586.51, or $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It gave a refundable CTC of $4,300 based only on the number of children, without subtracting the $4,586.51 tax that the nonrefundable credit covers first. The ACTC is only the $2,013.49 of the $6,600 credit left after that offset."
us,scenario_026,free_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model reversed the income comparison by calling $85,208.82 well below the free-meal standard. That income is 220% of the family-of-five poverty guideline, and without SNAP/TANF categorical eligibility or universal free meals in North Carolina, the household must pay full price."
us,scenario_026,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_026,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_026,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model got FICA on $85,209 right ($5,283.00 + $1,235.53) but then added $1,016.75 of North Carolina income tax and called it a state payroll tax. NC has no mandatory employee state payroll tax, so the correct total is FICA alone: $6,518.47."
-us,scenario_026,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"The model added the $7,746 overtime premium on top of the $85,209 gross wages, even though gross wages already include overtime. That gave a $92,955 base and about $7,112 of FICA. It then submitted $13,330, which matches no step in its own reasoning. The correct answer is 7.65% of $85,209, or $6,518.47."
-us,scenario_026,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"The model added the $7,746 FLSA overtime premium to the $85,209 gross wages and applied 7.65% FICA to $92,955. The prompt says gross wages already include overtime, so this double-counted overtime. The correct base is $85,209, giving $6,518.47."
-us,scenario_026,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model double-counted overtime with a $92,955 base. It then subtracted the $21,208 employer-sponsored insurance premiums as a pre-tax cafeteria-plan exclusion, but those premiums belong to the head, who has no wages, and they don't reduce the FICA base here. Applying 7.65% to the spouse's full $85,209 gives $6,518.47."
-us,scenario_026,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model said it applied 6.2% and 1.45% to $85,209 plus $7,746 of overtime, which double-counts overtime already included in gross wages. Its $6,732.19 also doesn't match its own stated base ($92,955 × 7.65% = $7,111.06). The correct figure is 7.65% of $85,209, or $6,518.47."
-us,scenario_026,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model subtracted the head's $21,208 employer-sponsored insurance premiums from the spouse's $85,209 wages and applied FICA to $64,001. Those premiums don't reduce the earner's Social Security or Medicare wages, so the full $85,209 is the base, giving $6,518.47."
-us,scenario_026,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly applied 6.2% Social Security and 1.45% Medicare to the spouse's $85,209 and correctly found no Additional Medicare Tax and no NC payroll tax. It then added the components wrong: $5,282.96 + $1,235.53 = $6,518.49, not $6,519.97, leaving it $1.50 off."
-us,scenario_026,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's $10,618 implies a FICA base of about $138,800, well above anything the household earns. The only earner has $85,209 in wages, overtime included, so 7.65% gives $6,518.47."
-us,scenario_026,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model treated total wages as $92,955 by adding the $7,746 overtime premium to the $85,209 gross wages. The prompt says gross wages already include overtime, so the FICA base is $85,209 and the tax is $6,518.47."
-us,scenario_026,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for payroll_tax, so there was no answer to score. The correct derivation is 7.65% employee FICA on the spouse's $85,209 in wages, or $6,518.47."
-us,scenario_026,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model said employer coverage and modest household income produce roughly zero employee payroll tax. Employee FICA has no income exemption and isn't offset by employer contributions, so the spouse owes 6.2% plus 1.45% on all $85,209 of wages, or $6,518.47."
+us,scenario_026,payroll_tax,claude-fable-5.2,llm_error,payroll_tax_base,False,"It took the $2,778 traditional 401(k) deferral out of the FICA base and taxed $82,431. Under IRC §3121(v)(1)(A), elective 401(k) deferrals stay in Social Security and Medicare wages, so 7.65% applies to the full $85,209, giving $6,518.47."
+us,scenario_026,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Its federal Social Security and Medicare amounts on $85,209 were right, but it then added a made-up North Carolina 'state payroll tax' of about $1,016.75. North Carolina has no mandatory employee payroll tax, and NC income tax is not part of this output, so the total should stop at $6,518.47."
+us,scenario_026,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"It added the $7,746 overtime premium to the $85,209 wages, even though the prompt says gross wages already include overtime. It then submitted $13,330, almost double its own $7,112 figure, which points to adding employer-side FICA or a second earner. The correct amount is 7.65% of $85,209 = $6,518.47."
+us,scenario_026,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It counted the overtime twice by taxing $85,209 + $7,746 = $92,955. The prompt says the gross wage figure already includes overtime, so the FICA base is $85,209 and the tax is $6,518.47."
+us,scenario_026,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It made two mistakes in the base. It added the $7,746 overtime premium on top of wages that already include it. It also subtracted the head's $21,208 employer-sponsored insurance premiums from the spouse's wages, but those premiums are listed for the head, who has no wages, so they don't reduce the spouse's FICA base. The correct base is $85,209, giving $6,518.47."
+us,scenario_026,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"It treated the $7,746 overtime premium as extra pay on top of the $85,209 wages, which double-counts overtime already in the gross figure. Its $6,732.19 also doesn't equal 7.65% of $85,209 or of $92,955, so it miscalculated as well. The correct amount is 7.65% of $85,209 = $6,518.47."
+us,scenario_026,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It subtracted the head's $21,208 employer-sponsored insurance premiums from the spouse's $85,209 wages. Those premiums are listed for the head, who has no wages, so they don't reduce the spouse's FICA base. The full $85,209 is taxed, giving $6,518.47."
+us,scenario_026,payroll_tax,gpt-5.4-mini,llm_error,other,False,"Its method was correct: 6.2% plus 1.45% on $85,209, with no Additional Medicare Tax and no NC payroll tax. The arithmetic was wrong, though. It reported $6,519.97 instead of $5,282.95 + $1,235.53 = $6,518.47, an extra $1.50."
+us,scenario_026,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"It gave $10,618 with no base shown. That equals 7.65% of about $138,800, far above the spouse's actual $85,209, so it inflated the wage base or added employer-side tax. Employee FICA on $85,209 is $6,518.47."
+us,scenario_026,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"It taxed $92,955 by adding the $7,746 overtime premium to the $85,209 gross wages, which already include overtime. That double-counts overtime; the correct FICA base is $85,209 and the tax is $6,518.47."
+us,scenario_026,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no payroll_tax value and gave no reasoning, so there was nothing to score. The correct value is 7.65% of the spouse's $85,209 wages = $6,518.47."
+us,scenario_026,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"It answered zero on the grounds that household income was 'modest' and the employer covered the head. Employee FICA has no income exemption: 6.2% Social Security and 1.45% Medicare apply from the first dollar of the spouse's $85,209 wages, giving $6,518.47."
us,scenario_026,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $21,208 employer-sponsored insurance premium from $85,209 of wages to produce $64,001, even though the school-meal income calculation in this case uses $85,209. That income is 220% of the federal poverty guideline, above the 185% reduced-price limit, and no categorical pathway applies."
us,scenario_026,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_026,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -2482,56 +2516,57 @@ us,scenario_027,local_income_tax,glm-5.2,parse_contract_failure,missing_output,F
us,scenario_027,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_027,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_027,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It correctly saw that Connecticut's categorical eligibility applies and that the gross test passes. It then still applied the 100% FPL net income test, which categorical eligibility waives. It also counted only the Head's medical costs. With net income of $2,118, the formula benefit is $0, so the household gets the $24 monthly minimum ($288 a year), not $0."
-us,scenario_027,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It found the household ineligible because net income of about $2,308 exceeds the $1,763 net limit. Connecticut's categorical eligibility waives the net income test for this household. When 30% of net income exceeds the $546 maximum, a two-person household gets the $24 monthly minimum allotment."
-us,scenario_027,snap,claude-haiku-4.5,llm_error,asset_resource,False,"It said $2,513 a month exceeds a limit of about $2,900, which contradicts itself. It also applied a $3,500 asset limit. Connecticut's categorical eligibility uses a 200% FPL gross limit (the household is at 1.44) and has no asset test. The household is eligible and gets the $24 monthly minimum."
-us,scenario_027,snap,claude-haiku-5.5,llm_error,asset_resource,False,"It ruled the household out because $48,000 in bank assets exceeds the SNAP resource limit. Connecticut's categorical eligibility through TANF non-cash benefits waives the asset test. The household is eligible, and the $24 monthly minimum allotment applies."
-us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,"It applied a $4,500 elderly/disabled asset limit to the $48,000 in bank assets. Connecticut's categorical eligibility removes the asset test. The household is eligible, and since its formula benefit is $0 it gets the $24 monthly minimum ($288 a year)."
-us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"It found the household ineligible over a $4,500 resource limit. Connecticut's categorical eligibility (gross income is 1.44 times FPL, under the 200% limit) waives the asset test. The household is eligible and gets the $24 monthly minimum allotment."
-us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,It denied the benefit because income exceeds the net income limit. Connecticut's categorical eligibility waives that test. A two-person household whose 30% contribution ($636) exceeds the $546 maximum still gets the $24 monthly minimum.
-us,scenario_027,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"It compared net income of about $2,308 with the $1,763 net limit and paid nothing. Connecticut's categorical eligibility waives the net income test. Because the formula benefit is $0, the two-person minimum of $24 a month ($288 a year) applies."
-us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"It passed the gross test but then failed the household on a $4,500 asset limit. Connecticut's categorical eligibility has no asset test. The household is eligible, and with a 30% contribution above the $546 maximum it gets the $24 monthly minimum."
-us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"It applied a $4,250 elderly/disabled resource limit. Connecticut's categorical eligibility waives the asset test. The household is eligible and gets the $24 monthly minimum allotment ($288 a year)."
-us,scenario_027,snap,claude-sonnet-5.5,llm_error,period_annualization,False,"It treated the $650 of annual medical costs as a monthly amount, giving a $615 monthly deduction and net income of $1,712. The engine's medical deduction is about $210 a month. It counts the Head's $500 of other medical costs and the $202.90 monthly Medicare Part B premium, less the $35 disregard, and leaves out over-the-counter costs. Net income is $2,118. That gives a 30% contribution of $636, above the $546 maximum, so the benefit is the $24 monthly minimum ($288 a year), not $32 a month."
-us,scenario_027,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It ruled the household out on a $4,250 asset limit. Connecticut's categorical eligibility through TANF non-cash benefits removes the asset test. The household is eligible for the $24 monthly minimum allotment."
-us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It passed the 200% gross test but then denied the benefit because net income exceeds the 100% FPL net limit. Categorical eligibility exempts the household from the net income test. With a $0 formula benefit, a two-person household gets the $24 monthly minimum."
-us,scenario_027,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,It denied SNAP because net income exceeds the two-person net limit. Connecticut's categorical eligibility waives the net income test. The household gets the $24 monthly minimum allotment ($288 a year).
-us,scenario_027,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It applied the $1,762.50 net income limit to net income of $2,313. That test is waived for households categorically eligible through Connecticut's TANF non-cash program. When the formula benefit is $0, a two-person household gets the $24 monthly minimum."
-us,scenario_027,snap,gemini-3-flash-preview,llm_error,other,False,"It correctly found categorical eligibility and a $0 formula benefit, then stopped. It skipped the minimum allotment, which guarantees eligible one- and two-person households $24 a month ($288 a year)."
-us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"It denied SNAP because $48,000 in assets exceeds the resource limit. Connecticut's categorical eligibility waives the asset test. The household is eligible and gets the $24 monthly minimum allotment."
-us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,other,False,"It saw that the household is categorically eligible and that 30% of net income exceeds the maximum, but it returned $0. It missed the minimum allotment: eligible two-person households get $24 a month, or $288 a year."
-us,scenario_027,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly applied categorical eligibility and the minimum allotment, but used the FY2025 minimum of $23. The 2026 minimum is $24 a month, so the annual total is $288, not $276."
-us,scenario_027,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Its $0 answer means it treated high net income or the assets as ruling the household out. Connecticut's categorical eligibility waives both the net income and asset tests, and eligible two-person households get the $24 monthly minimum ($288 a year)."
-us,scenario_027,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly found categorical eligibility with no asset limit and applied the minimum allotment. However, it used the outdated $23 figure instead of the 2026 minimum of $24 a month. The result is $276 instead of $288."
-us,scenario_027,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied the minimum allotment for a categorically eligible household at the FY2025 amount of $23 a month. The 2026 minimum is $24, giving $288 a year."
-us,scenario_027,snap,gemini-3.8-flash,llm_error,asset_resource,False,"It cited the $48,000 in assets as disqualifying and took the $0 formula benefit as final. Connecticut's categorical eligibility waives the asset test, and a two-person household with a $0 formula benefit gets the $24 monthly minimum."
-us,scenario_027,snap,gemini-4-argon,llm_error,categorical_eligibility,False,"It treated 100% FPL as the categorical eligibility limit and applied an asset limit. Connecticut's categorical eligibility uses a 200% FPL gross limit, which the household passes at 1.44, and has no asset test. The household is eligible and gets the $24 monthly minimum."
-us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,"It gave no SNAP value and no explanation, so no answer could be scored. The correct value is the $24 monthly minimum allotment times 12, or $288."
-us,scenario_027,snap,glm-5.3,llm_error,other,False,"It correctly found that 30% of net income exceeds the two-person maximum, though it used an outdated $536 maximum. It then returned $0 instead of applying the minimum allotment. A categorically eligible two-person household gets $24 a month ($288 a year)."
-us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,"It ruled the household out under typical asset and income rules. Connecticut's categorical eligibility waives the asset and net income tests, and gross income (1.44 times FPL) passes the 200% limit. The household gets the $24 monthly minimum allotment."
-us,scenario_027,snap,gpt-5.4-nano,llm_error,other,False,"It claimed key inputs were missing and defaulted to $0 instead of computing the benefit. The facts given produce net income of $2,118 and a $0 formula benefit, and the $24 monthly minimum for categorically eligible two-person households then applies ($288 a year)."
-us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,"It treated net income above the SNAP limit or phaseout as ending eligibility. Connecticut's categorical eligibility waives the net income test. When the formula benefit phases out to $0, a two-person household still gets the $24 monthly minimum."
-us,scenario_027,snap,gpt-5.6-luna,llm_error,other,False,"It took the $0 formula result after standard and medical deductions as the final benefit. It missed the minimum allotment, which pays categorically eligible one- and two-person households $24 a month ($288 a year)."
-us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,It denied SNAP because net income exceeds the two-person net limit. Connecticut's categorical eligibility exempts this household from the net income test. The $24 monthly minimum allotment applies.
-us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"It applied the elderly/disabled resource limit to the $48,000 in bank assets. Connecticut's categorical eligibility waives the asset test. The household gets the $24 monthly minimum ($288 a year)."
-us,scenario_027,snap,gpt-6-luna,llm_error,other,False,It concluded that countable income is too high for a positive allotment and stopped at $0. It missed the minimum allotment: eligible two-person households get $24 a month even when 30% of net income exceeds the $546 maximum.
-us,scenario_027,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It failed the household on the two-person net income limit. Connecticut's categorical eligibility waives that test. With a $0 formula benefit, the household gets the $24 monthly minimum allotment."
-us,scenario_027,snap,gpt-6.1-sol,llm_error,other,False,"It correctly found that the 30% contribution ($636) exceeds the $546 two-person maximum, then returned $0. It skipped the $24 monthly minimum for eligible one- and two-person households, which totals $288 a year."
-us,scenario_027,snap,gpt-6.1-terra,llm_error,other,False,"It treated net income too high for a formula allotment as meaning no benefit. Because Connecticut's categorical eligibility makes the household eligible, the two-person minimum of $24 a month applies, totaling $288."
-us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,"It cited assets above typical SNAP limits as disqualifying. Connecticut's categorical eligibility waives the asset test, and gross income (1.44 times FPL) passes the 200% limit. The household gets the $24 monthly minimum."
-us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"It applied a $4,500 elderly/disabled resource limit regardless of income. Connecticut's categorical eligibility removes the asset test. The household is eligible for the $24 monthly minimum allotment."
-us,scenario_027,snap,grok-4.6,llm_error,other,False,"It correctly found that 30% of net income exceeds the two-person maximum, but returned $0. It missed the minimum allotment, which pays eligible two-person households $24 a month ($288 a year)."
-us,scenario_027,snap,grok-4.7,llm_error,thresholds_rates,False,"It correctly applied categorical eligibility, the asset waiver, and the minimum allotment, but at the FY2025 amount of $23. The 2026 minimum is $24 a month, so the annual total is $288, not $276."
-us,scenario_027,snap,grok-build-0.1,llm_error,other,False,"It compared annual net income of about $30,000 with the annual maximum allotment and returned $0. Monthly net income is actually $2,118. The formula benefit is still $0, but it missed the $24 monthly minimum for categorically eligible two-person households."
-us,scenario_027,snap,inkling,llm_error,categorical_eligibility,False,"It ruled the household ineligible because net income of about $2,317 exceeds the 100% FPL net limit of about $1,762. Connecticut's categorical eligibility waives the net income test. The household gets the $24 monthly minimum allotment."
-us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,It found the household ineligible because net income exceeds the 100% FPL net limit. Categorical eligibility through Connecticut's TANF non-cash program exempts the household from that test. The $24 monthly minimum applies ($288 a year).
-us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,"It applied the two-person net income limit and denied SNAP. Connecticut's categorical eligibility waives the net income test. With a $0 formula benefit, the household gets the $24 monthly minimum allotment."
-us,scenario_027,snap,minimax-m3,llm_error,thresholds_rates,False,"It put the 200% FPL gross limit for two people at about $30,120 a year, which is far below the actual limit of about $42,300 ($3,525 a month). It also applied a $4,250 asset limit, which categorical eligibility waives. The household passes at 1.44 times FPL and gets the $24 monthly minimum."
-us,scenario_027,snap,muse-spark-1.3,llm_error,asset_resource,False,"It ruled the household out on a $4,250 elderly/disabled asset limit. Connecticut's categorical eligibility has no asset test. The household is eligible for the $24 monthly minimum allotment ($288 a year)."
-us,scenario_027,snap,muse-wonder,llm_error,categorical_eligibility,False,"It denied SNAP because net income exceeds the elderly/disabled net limit. Connecticut's categorical eligibility waives the net income test. When the formula benefit is $0, a two-person household gets the $24 monthly minimum."
-us,scenario_027,snap,ox-alpha,llm_error,categorical_eligibility,False,"It failed the household on the net income test and zeroed the benefit because 30% of net income exceeds the maximum. Categorical eligibility waives the net income test, and an eligible two-person household gets the $24 monthly minimum instead of $0."
-us,scenario_027,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"It claimed $2,536 a month exceeds the gross and net limits and that the assets disqualify the household. Connecticut's categorical eligibility uses a 200% FPL gross limit, which the household passes at 1.44, and waives both the net income and asset tests. The household gets the $24 monthly minimum."
-us,scenario_027,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It made up a $51,000 resource limit and claimed net income is below the poverty guideline, which led to $430 a month. Net income is $2,118, or 1.2 times FPL, so the 30% contribution of $636 wipes out the $546 maximum. Only the $24 monthly minimum is paid ($288 a year)."
+us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"It correctly found the household eligible under CT's 200% FPL BBCE limit, then still applied the 100% FPL net income test, which BBCE households are exempt from, and called the household ineligible. It also never applied the $24/month minimum allotment that an eligible two-person household gets when 30% of net income exceeds the $546 maximum."
+us,scenario_027,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"It ruled the household ineligible because its ~$2,308 net income exceeds the 100% FPL net limit. It ignored that CT's TANF non-cash BBCE waives the net income test. As an eligible two-person household, it gets the $24/month minimum allotment ($288/year)."
+us,scenario_027,snap,claude-fable-5.2,llm_error,thresholds_rates,False,"It correctly applied BBCE and the two-person minimum allotment, but used 8% of the $298 one-person maximum unrounded ($23.84). The FY2026 minimum allotment is rounded to $24/month, giving $288 rather than $286.08."
+us,scenario_027,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It applied a 130% FPL gross limit and the elderly/disabled asset limit, wrongly asserting that standard asset limits apply in Connecticut. CT's BBCE uses a 200% FPL gross limit with no asset test, and $2,536.25/month (1.44× poverty) passes. The household is eligible for the $24/month minimum allotment."
+us,scenario_027,snap,claude-haiku-5.5,llm_error,asset_resource,False,"It denied SNAP because $48,000 in bank assets exceeds the federal resource limit. Connecticut's BBCE waives the asset test for households under 200% FPL, so the household is eligible and gets the $24/month minimum allotment."
+us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,"It applied the ~$4,500 elderly/disabled asset limit to the $48,000 balance. Connecticut's TANF non-cash BBCE removes the asset test entirely. The household is categorically eligible and gets the $24/month minimum allotment."
+us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"It treated the $48,000 in assets as disqualifying under the ~$4,500 elderly/disabled resource limit. That limit does not apply because CT's BBCE grants categorical eligibility at 1.44× poverty. The result is the $24/month minimum allotment, not zero."
+us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,"It zeroed the benefit because income exceeds the two-person net income limit. CT's BBCE waives the net income test, and the household gets the $24/month minimum allotment even though the formula benefit is zero."
+us,scenario_027,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"It compared ~$2,308 net income to the ~$1,763 net income limit and denied benefits. Categorical eligibility through CT's TANF non-cash BBCE waives that test. The household gets the $24/month two-person minimum allotment, or $288/year."
+us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"It checked the 200% FPL gross test but then applied a $4,500 elderly/disabled asset limit to the $48,000 balance. CT's BBCE eliminates the asset test for households under 200% FPL, so the household is eligible for the $24/month minimum allotment."
+us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"It used the $4,250 elderly/disabled asset limit to deny eligibility, missing that Connecticut's BBCE waives the resource test. As a categorically eligible two-person household, it gets the $24/month minimum allotment."
+us,scenario_027,snap,claude-sonnet-5.5,llm_error,period_annualization,False,"It treated the $650 of annual medical and OTC expenses as monthly, producing a $615 excess medical deduction instead of about $19/month. That lowered net income to ~$1,712 and produced a $32/month formula benefit. Using the correct monthly figures, net income is $2,118, the formula benefit is zero, and only the $24/month minimum applies."
+us,scenario_027,snap,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It disqualified the household on the $4,250 elderly/disabled asset limit. Connecticut's TANF non-cash BBCE has no asset test, and with gross income at 1.44× poverty the household is eligible for the $24/month minimum allotment."
+us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"It noted gross income was within 200% FPL but still applied the 100% FPL net income test, which BBCE-eligible households are exempt from. It then returned zero instead of the $24/month minimum allotment for an eligible two-person household."
+us,scenario_027,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"It denied benefits on the net income limit. CT's BBCE waives the net income test, and an eligible two-person household with a zero formula benefit gets the $24/month minimum allotment ($288/year)."
+us,scenario_027,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"It compared $2,313 net income to the $1,762.50 100% FPL limit and set SNAP to zero. CT's TANF non-cash BBCE waives that net test, so the household is eligible for the $24/month minimum allotment."
+us,scenario_027,snap,gemini-3-flash-preview,llm_error,other,False,"It correctly found BBCE eligibility at 200% FPL and a zero formula benefit, but omitted the minimum allotment rule. Eligible one- and two-person households get at least $24/month, or $288/year."
+us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"It denied SNAP because $48,000 in assets exceeds the resource limit. Connecticut's BBCE waives the asset test, so the household is eligible and gets the $24/month two-person minimum allotment."
+us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,other,False,"It recognized categorical eligibility and that 30% of net income exceeds the $546 maximum, but returned zero. Eligible two-person households get the $24/month minimum allotment, which gives $288/year."
+us,scenario_027,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly applied BBCE and the minimum allotment, but used the outdated $23/month minimum. The FY2026 minimum is $24/month (8% of the $298 one-person maximum, rounded), so the total is $288 rather than $276."
+us,scenario_027,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned zero on income and assets. CT's BBCE waives the asset and net income tests at 1.44× poverty, and the household gets the $24/month minimum allotment even though the formula benefit is zero."
+us,scenario_027,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It correctly identified BBCE eligibility and the minimum allotment, but used the prior-year $23/month figure. The FY2026 two-person minimum allotment is $24/month, or $288/year."
+us,scenario_027,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied the minimum allotment rule with the outdated $23/month amount. For 2026 the minimum is $24/month (8% of the $298 one-person maximum, rounded), giving $288."
+us,scenario_027,snap,gemini-3.8-flash,llm_error,asset_resource,False,"It applied an asset limit that CT's BBCE waives, and it stopped at a zero formula benefit. An eligible two-person household gets the $24/month minimum allotment."
+us,scenario_027,snap,gemini-4-argon,llm_error,categorical_eligibility,False,"It claimed income above 100% FPL defeats categorical eligibility and then applied the elderly/disabled asset limit. CT's BBCE threshold is 200% FPL, so the household at 1.44× poverty is categorically eligible with no asset test and gets the $24/month minimum."
+us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,"It submitted no SNAP value and no explanation, so there is no answer to score against the $288 reference."
+us,scenario_027,snap,glm-5.3,llm_error,other,False,"It correctly found that 30% of ~$2,300 net income exceeds the two-person maximum (it used an outdated $536 instead of $546), but returned zero. The household is categorically eligible under CT BBCE and gets the $24/month minimum allotment."
+us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,"It treated the $48,000 in bank assets and the SSDI income as disqualifying under typical limits. Connecticut's BBCE waives the asset test and uses a 200% FPL gross limit, so the household is eligible for the $24/month minimum allotment."
+us,scenario_027,snap,gpt-5.4-nano,llm_error,other,False,"It defaulted to zero, citing missing shelter and expense inputs, instead of computing eligibility. The household passes CT's BBCE gross test and, with no shelter costs, gets the $24/month minimum allotment."
+us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,"It concluded net income exceeds the SNAP limit and phaseout. CT's BBCE waives the net income test, and the two-person minimum allotment of $24/month applies when the formula benefit is zero."
+us,scenario_027,snap,gpt-5.6-luna,llm_error,other,False,It computed a zero formula benefit after standard and medical deductions but never applied the minimum allotment. Categorically eligible two-person households get $24/month.
+us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It denied SNAP because net income exceeds the two-person net limit. CT's TANF non-cash BBCE waives the net income test, which leaves the household eligible for the $24/month minimum allotment."
+us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"It applied the elderly/disabled resource limit to $48,000 in bank assets. Connecticut's BBCE has no asset test, so the household is categorically eligible and gets the $24/month minimum."
+us,scenario_027,snap,gpt-6-luna,llm_error,other,False,"It stopped at a zero allotment because countable income is too high, omitting the rule that eligible one- and two-person households get the $24/month minimum allotment. That gives $288/year."
+us,scenario_027,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"It treated the two-person net income limit as binding. BBCE eligibility in Connecticut waives that test, and the household gets the $24/month minimum allotment."
+us,scenario_027,snap,gpt-6.1-sol,llm_error,other,False,It correctly found that the 30% benefit reduction exceeds the $546 maximum but returned zero. An eligible two-person household gets the $24/month minimum allotment.
+us,scenario_027,snap,gpt-6.1-terra,llm_error,categorical_eligibility,False,"It concluded net income is too high for an allotment. CT's BBCE waives the net income test, and the two-person minimum of $24/month applies even when the formula benefit is zero."
+us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,"It cited assets above typical SNAP limits. Connecticut's BBCE removes the asset test, and gross income at 1.44× poverty passes the 200% FPL limit, so the household gets the $24/month minimum."
+us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"It treated the ~$4,500 elderly/disabled resource limit as disqualifying regardless of income. CT's TANF non-cash BBCE waives the asset test, so the household is eligible for the $24/month minimum allotment."
+us,scenario_027,snap,grok-4.6,llm_error,other,False,It correctly computed that 30% of net income exceeds the two-person maximum but set the benefit to zero. Categorically eligible one- and two-person households get the $24/month minimum allotment.
+us,scenario_027,snap,grok-4.7,llm_error,thresholds_rates,False,"It correctly applied BBCE and the minimum allotment, but used the outdated $23/month minimum instead of the FY2026 $24/month, so it reported $276 instead of $288."
+us,scenario_027,snap,grok-build-0.1,llm_error,other,False,"It compared roughly $30,000 of annual net income directly to the annual maximum allotment, then returned zero. It never applied CT's BBCE eligibility or the $24/month minimum allotment for an eligible two-person household."
+us,scenario_027,snap,inkling,llm_error,categorical_eligibility,False,"It denied eligibility because ~$2,317 net income exceeds 100% FPL. CT's BBCE waives the net income test, and the household gets the $24/month minimum allotment."
+us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It applied the 100% FPL net income limit and found the household ineligible. Categorical eligibility through Connecticut's TANF non-cash BBCE waives that test, and the household gets the $24/month two-person minimum."
+us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,"It treated the two-person net income limit as disqualifying. CT's BBCE waives the net income test, so the household is eligible and gets the $24/month minimum allotment."
+us,scenario_027,snap,minimax-m3,llm_error,thresholds_rates,False,"It misstated the 200% FPL gross limit for two as ~$30,120/year; the correct figure is $3,525/month (~$42,300/year), which $30,435 passes. It also applied the $4,250 asset limit that CT's BBCE waives, so it missed the $24/month minimum allotment."
+us,scenario_027,snap,muse-spark-1.3,llm_error,asset_resource,False,"It disqualified the household on the ~$4,250 elderly/disabled asset limit. Connecticut's BBCE eliminates the asset test, so the household is categorically eligible for the $24/month minimum allotment."
+us,scenario_027,snap,muse-wonder,llm_error,categorical_eligibility,False,"It applied the elderly/disabled net income limit and found the household ineligible. CT's TANF non-cash BBCE waives the net income test, and the household gets the $24/month minimum allotment."
+us,scenario_027,snap,ox-alpha,llm_error,categorical_eligibility,False,"It treated the net income test as failed and the zero formula benefit as final. BBCE exempts the household from the net test, and as an eligible two-person household it gets the $24/month minimum allotment."
+us,scenario_027,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"It claimed income exceeds the SNAP gross and net limits and that assets exceed the resource limit. CT's BBCE uses a 200% FPL gross limit with no asset or net test, so the household is eligible for the $24/month minimum allotment."
+us,scenario_027,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It invented a $51,000 resource limit and claimed net income was below poverty, then awarded a large $430/month allotment. Net income is $2,118/month, and 30% of that ($636) exceeds the $546 maximum, so only the $24/month minimum applies."
us,scenario_027,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_027,spouse_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model correctly saw that $30,435 exceeds the 2-person 138% FPL threshold. It then cut MAGI to $275 by counting only taxable Social Security, but Medicaid MAGI adds back non-taxable Social Security benefits. Household MAGI stays at $30,435 (141% FPL), which is above the expansion limit."
us,scenario_027,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model treated the spouse as qualifying for blind/disabled Medicaid because the spouse has no income of their own. ABD Medicaid counts the couple's combined resources, and the $48,000 in bank assets is far above Connecticut's ABD resource limit and the SSI couple limit. The ABD pathway fails, SSI is $0, and the 141% FPL MAGI is above the 138% expansion limit."
@@ -2564,27 +2599,28 @@ us,scenario_027,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wron
us,scenario_027,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_027,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_027,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_028,child1_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"Concluded eligible only because income of about 185% FPL falls between the Medicaid ceiling and the ~314% FPL CHIP ceiling. Never applied the rest of CHIP's own child test. PolicyEngine fails that test for this child, and the child's employer-sponsored insurance excludes them as a targeted low-income child, so the answer is 0."
-us,scenario_028,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"Explicitly ruled that other coverage does not affect PA CHIP eligibility, and also overstated income at about 240% FPL when it is about 186%. CHIP's own child test fails in PolicyEngine. A child covered by employer-sponsored insurance is not a targeted low-income child, so eligibility is 0."
-us,scenario_028,child1_chip_eligible,claude-opus-5,llm_error,health_coverage,False,"Applied only the under-19 age test and a 190% versus 314% FPL income comparison. Ignored that the child's employer-sponsored insurance makes CHIP's own eligibility test fail in PolicyEngine. Medicaid is denied and the CHIP child test is false, so the correct value is 0."
-us,scenario_028,child1_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"Reasoned that 187% FPL sits between the Medicaid limit and the 314% CHIP limit, which handles only the Medicaid-gap income band. Skipped CHIP's own child test, which PolicyEngine fails for this ESI-covered child, so eligibility is 0, not 1."
-us,scenario_028,child1_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"Equated CHIP eligibility with income of about 186% FPL falling below the ~314% PA limit. Disregarded the listed employer-sponsored insurance, which excludes the child from targeted low-income child status. PolicyEngine's CHIP child test fails, so the answer is 0."
-us,scenario_028,child1_chip_eligible,deepseek-v4-flash-0731,llm_error,health_coverage,False,"Placed income at 190–200% FPL inside an assumed 158–314% PA CHIP band and stopped there. Never applied the requirement that a CHIP child not already be covered by a group health plan. This child has ESI and fails PolicyEngine's CHIP child test, so the value is 0."
-us,scenario_028,child1_chip_eligible,deepseek-v4.1-flash,llm_error,health_coverage,False,"Compared $60,010 with an assumed 300% FPL cutoff of $99,000 and treated age plus income as the whole test. PolicyEngine's CHIP child criteria fail for this child, who is covered by employer-sponsored insurance, so the correct output is 0."
-us,scenario_028,child1_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"Treated 'under 19 and below 314% FPL' as sufficient for PA CHIP. The child's employer-sponsored insurance removes targeted low-income child status, and PolicyEngine's CHIP child test returns false, so eligibility is 0."
-us,scenario_028,child1_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"Said only that the child is under 19 and within PA CHIP income limits. Never applied the rest of CHIP's own child test, which PolicyEngine fails for this ESI-covered child. With Medicaid also denied, CHIP eligibility is 0."
-us,scenario_028,child1_chip_eligible,gemini-3.8-flash,llm_error,health_coverage,False,"Relied on age under 19 and a generic statement that income is within PA CHIP limits. Ignored the child's employer-sponsored insurance, under which PolicyEngine's CHIP child test fails, so the correct value is 0."
-us,scenario_028,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"Returned no value and no explanation for child1_chip_eligible, so there was no answer to score. The correct output is 0: the child is not Medicaid-eligible and fails CHIP's own child test."
-us,scenario_028,child1_chip_eligible,glm-5.3,llm_error,health_coverage,False,"Concluded eligible because 185% FPL is above the child Medicaid threshold and below the PA CHIP ceiling, treating CHIP as a pure income band. Missed that the child's ESI coverage makes CHIP's own child test fail in PolicyEngine, so the value is 0."
-us,scenario_028,child1_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"Asserted the child is within PA's higher CHIP income range and called that eligibility. Did not apply the CHIP child test that PolicyEngine fails for this ESI-covered child, so the correct value is 0."
-us,scenario_028,child1_chip_eligible,grok-4.6,llm_error,health_coverage,False,"Correctly found the child is not Medicaid-eligible, then treated MAGI below 314% FPL as the only remaining CHIP condition. PolicyEngine's CHIP child test fails for this child, who already has employer-sponsored insurance, so eligibility is 0."
-us,scenario_028,child1_chip_eligible,grok-4.7,llm_error,health_coverage,False,"Correctly denied Medicaid, then declared CHIP eligibility from age and MAGI below 314% FPL alone. Ignored that a child covered by employer-sponsored insurance fails CHIP's own child criteria, which PolicyEngine returns as false, so the value is 0."
-us,scenario_028,child1_chip_eligible,kimi-k2.6,llm_error,health_coverage,False,"Compared MAGI of $60,010 for a family of four against the 314% FPL limit and stopped. Skipped the CHIP child requirement that PolicyEngine fails for this ESI-covered child, so the answer is 0."
-us,scenario_028,child1_chip_eligible,kimi-k3,llm_error,health_coverage,False,"Explicitly asserted that current ESI coverage does not negate CHIP eligibility. That is the error: a child covered by a group health plan is not a targeted low-income child, and PolicyEngine's CHIP child test fails, so the value is 0."
-us,scenario_028,child1_chip_eligible,minimax-m3,llm_error,health_coverage,False,"Placed $61,277 inside an assumed 138%–319% FPL CHIP range for a family of four and called the child eligible. Ignored the child's employer-sponsored insurance, under which PolicyEngine's CHIP child test fails, so the correct value is 0."
-us,scenario_028,child1_chip_eligible,muse-spark-1.3,llm_error,health_coverage,False,"Judged eligibility only from income of about 186% FPL against an assumed 300% FPL ceiling. Missed that the ESI-covered child fails CHIP's own child test in PolicyEngine, so the value is 0."
-us,scenario_028,child1_chip_eligible,ox-alpha,llm_error,health_coverage,False,"Computed MAGI correctly at about 180–190% FPL, but explicitly dismissed the employer-sponsored insurance as 'enrollment status, not eligibility.' CHIP excludes children already covered by a group health plan, and PolicyEngine's CHIP child test fails, so eligibility is 0."
-us,scenario_028,child1_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"Asserted only that the child is within PA's CHIP income range, treating CHIP as an income-band test. Ignored the child's employer-sponsored insurance, under which PolicyEngine's CHIP child test fails, so the correct value is 0."
+us,scenario_028,child1_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"The model based eligibility only on income being about 185% FPL, between the Medicaid limit and PA's CHIP ceiling. It ignored that Child 1 has employer-sponsored insurance, so the child fails PolicyEngine's CHIP child test and is not CHIP-eligible."
+us,scenario_028,child1_chip_eligible,claude-fable-5.2,llm_error,health_coverage,False,"The model explicitly assumed that existing employer coverage does not bar CHIP eligibility. That is the exact error: CHIP excludes a child covered by a group health plan, so Child 1's ESI makes the CHIP child test fail even though the age test is met."
+us,scenario_028,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model stated that PA CHIP eligibility depends on income, not on other coverage. It also overstated income at about 240% FPL. The rule it missed is that CHIP excludes a child with employer-sponsored group coverage, which makes Child 1 fail PolicyEngine's CHIP child test."
+us,scenario_028,child1_chip_eligible,claude-opus-5,llm_error,health_coverage,False,"The model treated being under 19 with income of about 190% FPL as enough for CHIP. It ignored that Child 1 has employer-sponsored insurance, which fails CHIP's own child criteria, and the child has no Medicaid pathway."
+us,scenario_028,child1_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"The model concluded eligibility only because income of about 187% FPL falls between the Medicaid and CHIP limits. It never applied CHIP's exclusion of children covered by an employer group health plan, which Child 1 has, so the CHIP child test fails."
+us,scenario_028,child1_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"The model decided eligibility only by placing income of about 186% FPL between the Medicaid limit and PA's 314% CHIP limit. It overlooked Child 1's employer-sponsored insurance, which disqualifies the child under PolicyEngine's CHIP child test."
+us,scenario_028,child1_chip_eligible,deepseek-v4-flash-0731,llm_error,health_coverage,False,"The model checked only that income of roughly 190–200% FPL falls within a 158–314% CHIP band. It ignored that Child 1 is covered by employer-sponsored insurance, which fails CHIP's own child criteria, so is_chip_eligible is False."
+us,scenario_028,child1_chip_eligible,deepseek-v4.1-flash,llm_error,health_coverage,False,"The model applied only an age test and an income test against an assumed 300% FPL ($99,000) limit. It skipped the CHIP requirement that the child not be covered by a group health plan; Child 1 has ESI, so the child fails the CHIP child test."
+us,scenario_028,child1_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"The model treated CHIP as available to any child under 19 in a household below 314% FPL. It ignored Child 1's employer-sponsored insurance, which makes the child fail PolicyEngine's CHIP child test, and the child has no Medicaid route."
+us,scenario_028,child1_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"The model gave only age and a general income-within-limits check. It never applied CHIP's exclusion for children with employer group coverage, which Child 1 has, so the CHIP child test fails and the answer is No."
+us,scenario_028,child1_chip_eligible,gemini-3.8-flash,llm_error,health_coverage,False,"The model relied only on Child 1 being under 19 with income within PA's CHIP range. It ignored the child's employer-sponsored insurance, which fails CHIP's own child criteria, so the child is not CHIP-eligible."
+us,scenario_028,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for child1_chip_eligible, so there is no answer to score. This is a missing-output failure, not a substantive one; the correct value is 0 because Child 1 fails the CHIP child test and is not Medicaid-eligible."
+us,scenario_028,child1_chip_eligible,glm-5.3,llm_error,health_coverage,False,The model correctly placed Child 1 above the Medicaid children's threshold but then granted CHIP based on age and income of about 185% FPL alone. It missed that Child 1's employer-sponsored insurance disqualifies the child under PolicyEngine's CHIP child test.
+us,scenario_028,child1_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"The model concluded eligibility from age 10 and income in Pennsylvania's higher CHIP range. It never accounted for Child 1's employer-sponsored insurance, which makes the child fail CHIP's own child criteria."
+us,scenario_028,child1_chip_eligible,grok-4.6,llm_error,health_coverage,False,"The model correctly found Child 1 not Medicaid-eligible but then treated MAGI below about 314% FPL as the only remaining CHIP condition. It ignored that the child is covered by employer-sponsored insurance, which fails PolicyEngine's CHIP child test."
+us,scenario_028,child1_chip_eligible,grok-4.7,llm_error,health_coverage,False,"The model correctly ruled out Medicaid but tested CHIP only on age and MAGI against the 314% FPL limit. It omitted CHIP's bar on children covered by an employer group health plan, which applies to Child 1, so CHIP eligibility is False."
+us,scenario_028,child1_chip_eligible,kimi-k2.6,llm_error,health_coverage,False,"The model compared MAGI of $60,010 with PA's 314% FPL CHIP limit and stopped there. It never applied the requirement that a CHIP child not have group health coverage; Child 1 has employer-sponsored insurance, so the CHIP child test fails."
+us,scenario_028,child1_chip_eligible,kimi-k3,llm_error,health_coverage,False,"The model explicitly asserted that current ESI coverage does not negate CHIP eligibility. That is backwards: CHIP excludes a child covered by a group health plan, so Child 1's employer-sponsored insurance makes the CHIP child test fail."
+us,scenario_028,child1_chip_eligible,minimax-m3,llm_error,health_coverage,False,"The model placed $61,277 inside a 138%–319% FPL CHIP band and declared Child 1 eligible on income alone. It ignored the child's employer-sponsored insurance, which disqualifies the child from CHIP under PolicyEngine's child criteria."
+us,scenario_028,child1_chip_eligible,muse-spark-1.3,llm_error,health_coverage,False,"The model used only age and income of about 186% FPL against an assumed 300% FPL limit. It skipped CHIP's exclusion of children with employer group coverage, which Child 1 has, so the child is not CHIP-eligible."
+us,scenario_028,child1_chip_eligible,ox-alpha,llm_error,health_coverage,False,"The model explicitly dismissed Child 1's employer-sponsored insurance as reflecting 'enrollment status, not eligibility.' That is the exact error: a CHIP child must not be covered by a group health plan, so the ESI makes Child 1 fail PolicyEngine's CHIP child test regardless of the MAGI-to-FPL ratio."
+us,scenario_028,child1_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"The model asserted only that Child 1 falls within Pennsylvania's CHIP income range. It ignored that the child has employer-sponsored insurance, which fails CHIP's own child criteria, and that the child has no Medicaid pathway, so the correct value is 0."
us,scenario_028,child1_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,child1_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied an approximately 215% FPL threshold as Pennsylvania's Medicaid limit for children. The engine places this child at 1.82 times FPL but in no Medicaid category, so the model used a broader child-coverage threshold instead of the applicable Medicaid pathway thresholds."
@@ -2597,28 +2633,29 @@ us,scenario_028,child1_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligi
us,scenario_028,child1_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model provided no output or explanation for child1_wic_eligible, violating the required structured-output contract."
us,scenario_028,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly placed a 10-year-old within WIC's eligible child age range. Children qualify categorically only while under age 5, so Child 1 is not eligible."
us,scenario_028,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model misapplied WIC's child age limit by treating age 10 as within it. Because WIC child eligibility ends at age 5, the income test cannot make this child eligible."
-us,scenario_028,child2_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"It concluded eligible only because income of about 185% FPL fell inside a 314% PA CHIP band. It never applied PolicyEngine's own CHIP child criteria, which fail for child2. It also ignored that child2 has employer-sponsored insurance, which disqualifies a targeted low-income child from CHIP."
-us,scenario_028,child2_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"It said outright that CHIP eligibility holds 'regardless of current ESI enrollment' and relied only on income of about 192% FPL being under 314%. PolicyEngine's CHIP child test fails for child2, and the child's group-health (ESI) coverage counts against CHIP's targeted low-income child definition."
-us,scenario_028,child2_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"It treated age under 19 plus income of about 187% FPL as enough, saying the child is eligible 'regardless of current enrollment'. It skipped CHIP's own child criteria, which fail in PolicyEngine for this ESI-covered 10-year-old, so is_chip_eligible is False."
-us,scenario_028,child2_chip_eligible,claude-opus-5,llm_error,health_coverage,False,"It checked only that the child is under 19 with income of about 190% FPL, inside PA's CHIP range. It never evaluated the remaining CHIP child criteria, including the child's employer-sponsored coverage, which make child2 fail PolicyEngine's CHIP test."
-us,scenario_028,child2_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"It put eligibility in the gap between Medicaid and the 314% CHIP limit and treated that income band as decisive. PolicyEngine's CHIP child criteria fail for child2, who also holds employer-sponsored insurance, so the correct answer is not eligible."
-us,scenario_028,child2_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"It answered eligible only because income of about 186% FPL sits between the Medicaid limit and the 314% CHIP limit. It ignored the CHIP child criteria that child2 fails in the engine, including the child's existing employer-sponsored coverage."
-us,scenario_028,child2_chip_eligible,deepseek-v4-flash-0731,llm_error,health_coverage,False,"It relied only on household income falling in PA's 158–314% FPL CHIP range. It did not apply CHIP's own child criteria, which child2 fails in PolicyEngine and which exclude a child already covered by employer-sponsored insurance."
-us,scenario_028,child2_chip_eligible,deepseek-v4.1-flash,llm_error,health_coverage,False,"It reduced CHIP to two tests: age 10, and income of $60,010 below a 300% FPL cutoff. That skipped the CHIP child criteria the engine fails for child2, including disqualification for an ESI-covered child."
-us,scenario_028,child2_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"It treated age under 19 and income of $60,010 below 314% FPL as sufficient. It never applied the rest of PolicyEngine's CHIP child test, which fails for child2, a child already enrolled in employer-sponsored insurance."
-us,scenario_028,child2_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"It cited only age under 19 and income within PA CHIP limits. It ignored the CHIP child criteria that fail for child2 in the engine, including the exclusion for children with group-health (ESI) coverage, so is_chip_eligible is False."
-us,scenario_028,child2_chip_eligible,gemini-3.8-flash,llm_error,health_coverage,False,"It answered eligible from age under 19 and income within PA's CHIP limit alone. It did not apply the CHIP child criteria that child2 fails in PolicyEngine, where the child's existing employer-sponsored coverage blocks targeted low-income child status."
-us,scenario_028,child2_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"No value or explanation was returned for child2_chip_eligible, so there was no answer to score. The correct output is 0 because child2 fails PolicyEngine's CHIP child criteria and is not Medicaid-eligible."
-us,scenario_028,child2_chip_eligible,glm-5.3,llm_error,health_coverage,False,"It placed the child above the Medicaid children's threshold and within PA's CHIP income limit at about 185% FPL, then stopped there. It never applied the CHIP child criteria that child2 fails in the engine, including the exclusion for children already covered by employer-sponsored insurance."
-us,scenario_028,child2_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"It relied on age 10 and income inside PA's higher CHIP range. It skipped the CHIP child criteria that child2 fails in PolicyEngine, which rule out a child with existing employer-sponsored coverage, so eligibility is 0."
-us,scenario_028,child2_chip_eligible,grok-4.6,llm_error,health_coverage,False,"It correctly found that child2 is not Medicaid-eligible, then wrongly assumed age plus MAGI below 314% FPL completes the CHIP test. It ignored the CHIP child criteria the engine fails for child2, including disqualification of a child already covered by ESI."
-us,scenario_028,child2_chip_eligible,grok-4.7,llm_error,health_coverage,False,"It noted Medicaid ineligibility and checked only age and MAGI against a 314% FPL limit. PolicyEngine's CHIP child criteria fail for child2, who has employer-sponsored coverage that excludes targeted low-income child status, so the answer is 0."
-us,scenario_028,child2_chip_eligible,kimi-k2.6,llm_error,health_coverage,False,"It treated MAGI of $60,010 below 314% FPL plus age 10 as meeting every CHIP rule. It never applied the CHIP child criteria that child2 fails in the engine, including the exclusion for children with employer-sponsored insurance."
-us,scenario_028,child2_chip_eligible,kimi-k3,llm_error,health_coverage,False,"It said explicitly that 'current ESI coverage does not negate eligibility' and relied on income of about 187% FPL sitting below a 319% ceiling. Under PolicyEngine's CHIP child criteria, child2 fails, and group-health (ESI) coverage is disqualifying for a targeted low-income child."
-us,scenario_028,child2_chip_eligible,minimax-m3,llm_error,health_coverage,False,"It wrongly added the $1,267 of child support to income, getting $61,277. It then treated any income in a 138–319% FPL band as qualifying. It never applied the CHIP child criteria that child2 fails in PolicyEngine, including the exclusion for an ESI-covered child."
-us,scenario_028,child2_chip_eligible,muse-spark-1.3,llm_error,health_coverage,False,"It checked only age 10 and income of about 186% FPL against an assumed 300% FPL cap. It skipped the CHIP child criteria that child2 fails in the engine, where existing employer-sponsored coverage rules the child out."
-us,scenario_028,child2_chip_eligible,ox-alpha,llm_error,health_coverage,False,"It stated that 'current employer-sponsored coverage does not negate eligibility under PolicyEngine rules' and relied on income of about 185% FPL being below the PA threshold. PolicyEngine's CHIP child criteria fail for child2, and ESI coverage disqualifies a targeted low-income child, so the value is 0."
-us,scenario_028,child2_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"It gave only an income-range justification. It never applied PolicyEngine's remaining CHIP child criteria, which child2 fails as an employer-insured 10-year-old who is also not Medicaid-eligible, so the correct answer is not eligible."
+us,scenario_028,child2_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"The model checked only the income band: about 185% FPL is above PA's Medicaid limit and below the 314% CHIP cap. It never applied CHIP's own rule excluding a child already covered by a group health plan. Child 2 has employer-sponsored insurance, so the CHIP child test fails even though age and income pass."
+us,scenario_028,child2_chip_eligible,claude-fable-5.2,llm_error,health_coverage,False,"The model said outright that it assumed existing employer coverage does not bar eligibility, and that assumption is the error. CHIP covers only targeted low-income children with no group health plan coverage, so Child 2's ESI fails CHIP's own criteria even though income (about 182% FPL) is inside PA's CHIP band."
+us,scenario_028,child2_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model treated CHIP eligibility as applying 'regardless of current ESI enrollment' and tested only age and income (about 192% FPL against 314%). CHIP's targeted low-income child definition excludes a child covered by employer-sponsored insurance, so Child 2 is not eligible."
+us,scenario_028,child2_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model concluded eligibility from age 10 and about 187% FPL being under PA's 314% threshold. It never checked that Child 2 already has employer-sponsored insurance, which fails CHIP's own requirement that the child have no group health plan coverage."
+us,scenario_028,child2_chip_eligible,claude-opus-5,llm_error,health_coverage,False,The model applied only the under-19 age test and the income test (about 190% FPL) and skipped CHIP's exclusion for children with group health coverage. Child 2's employer-sponsored insurance disqualifies them from CHIP.
+us,scenario_028,child2_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"The model placed income at about 187% FPL, between the Medicaid limit and PA's 314% CHIP cap, and stopped there. It ignored that Child 2 has employer-sponsored insurance, which fails CHIP's own child eligibility test."
+us,scenario_028,child2_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"The model's eligibility finding rests only on income (about 186% FPL) falling between PA's Medicaid and CHIP limits. It missed CHIP's requirement that the child not be covered by a group health plan, which Child 2's employer-sponsored insurance violates."
+us,scenario_028,child2_chip_eligible,deepseek-v4-flash-0731,llm_error,health_coverage,False,"The model placed household income inside a 158–314% FPL CHIP range and declared Child 2 eligible. It never applied CHIP's exclusion of children who already have employer-sponsored group coverage, and Child 2 has such coverage."
+us,scenario_028,child2_chip_eligible,deepseek-v4.1-flash,llm_error,health_coverage,False,"The model compared $60,010 only to a 300% FPL ($99,000) cutoff and treated that as sufficient. It never considered that Child 2 has employer-sponsored insurance, which fails CHIP's own no-group-coverage requirement."
+us,scenario_028,child2_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"The model treated being under 19 with income below 314% FPL as the full CHIP test. It skipped the requirement that a CHIP child have no group health plan coverage, and Child 2 has employer-sponsored insurance."
+us,scenario_028,child2_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"The model checked only age (under 19) and PA's CHIP income limit. It overlooked that Child 2 is covered by employer-sponsored insurance, which fails CHIP's own child eligibility criteria."
+us,scenario_028,child2_chip_eligible,gemini-3.8-flash,llm_error,health_coverage,False,"The model applied only the age and income tests. It did not apply CHIP's exclusion for a child already covered by an employer group health plan, and Child 2's listed ESI makes them ineligible."
+us,scenario_028,child2_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no value or explanation for child2_chip_eligible, so there is no substantive answer to score. The correct derivation is: Child 2 passes the age test and is not Medicaid-eligible, but already has employer-sponsored insurance and therefore fails CHIP's own criteria, giving 0."
+us,scenario_028,child2_chip_eligible,glm-5.3,llm_error,health_coverage,False,"The model correctly found income of about 185% FPL above the children's Medicaid limit and inside PA's CHIP band. It then stopped without applying CHIP's no-group-coverage requirement, which Child 2 fails because of employer-sponsored insurance."
+us,scenario_028,child2_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"The model treated age 10 plus income inside PA's CHIP range as sufficient. It never accounted for Child 2's employer-sponsored insurance, which fails CHIP's targeted low-income child definition."
+us,scenario_028,child2_chip_eligible,grok-4.6,llm_error,health_coverage,False,"The model correctly found that Child 2 is not Medicaid-eligible and that MAGI is below 314% FPL. It then treated those two facts as sufficient and missed CHIP's own exclusion for children covered by employer-sponsored insurance, which Child 2 has."
+us,scenario_028,child2_chip_eligible,grok-4.7,llm_error,health_coverage,False,"The model declared eligibility 'under income and age rules' alone. CHIP's own criteria also require that the child have no group health plan coverage, and Child 2's employer-sponsored insurance fails that requirement."
+us,scenario_028,child2_chip_eligible,kimi-k2.6,llm_error,health_coverage,False,"The model compared MAGI of $60,010 to PA's 314% FPL limit and stopped. It never applied CHIP's bar on children already covered by employer-sponsored group insurance, which applies to Child 2."
+us,scenario_028,child2_chip_eligible,kimi-k3,llm_error,health_coverage,False,"The model said outright that current ESI coverage does not negate eligibility, and that is the error. CHIP's targeted low-income child definition excludes a child covered by a group health plan, so Child 2's ESI fails CHIP even at about 187% FPL."
+us,scenario_028,child2_chip_eligible,minimax-m3,llm_error,health_coverage,False,"The model counted child support in income ($61,277) and checked only the 138–319% FPL band. Even if the band holds, it missed CHIP's requirement that the child have no group health plan coverage, which Child 2's employer-sponsored insurance violates."
+us,scenario_028,child2_chip_eligible,muse-spark-1.3,llm_error,health_coverage,False,"The model treated age 10 and about 186% FPL under a 300% limit as the whole test. It omitted CHIP's exclusion for children already covered by employer-sponsored insurance, which disqualifies Child 2."
+us,scenario_028,child2_chip_eligible,ox-alpha,llm_error,health_coverage,False,"The model stated that current employer-sponsored coverage does not negate eligibility under PolicyEngine rules, and that is the error. CHIP's own child criteria exclude a child covered by a group health plan, so Child 2 fails despite being under 19 at about 185% FPL."
+us,scenario_028,child2_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"The model relied only on income falling within PA's CHIP range. It ignored that Child 2 has employer-sponsored insurance, which fails CHIP's requirement that an eligible child have no group health plan coverage."
us,scenario_028,child2_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,child2_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied an incorrect 215% FPL Medicaid threshold for Pennsylvania children. At the engine’s 1.82-times-FPL income level, Child 2 exceeds every applicable Medicaid threshold and receives no Medicaid category."
@@ -2631,27 +2668,28 @@ us,scenario_028,child2_medicare_eligible,glm-5.2,parse_contract_failure,missing_
us,scenario_028,child2_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable child2_wic_eligible output, violating the required submission contract."
us,scenario_028,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 10 as within WIC's child age category; WIC child eligibility ends at age five. It also failed to apply the 185%-of-poverty income ceiling, which this household's approximately $61,277 income exceeds."
us,scenario_028,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly stated that a 10-year-old is within WIC's child age limit, which ends at age five. It also incorrectly classified the household as income-eligible despite approximately $61,277 exceeding the applicable 185%-of-poverty ceiling."
-us,scenario_028,child3_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"The model correctly placed income at about 185% FPL, between the Medicaid and CHIP limits, but treated that as enough for eligibility. It ignored the listed employer-sponsored insurance. Group health coverage disqualifies a child from being a targeted low-income child (42 CFR 457.310(b)(2)(ii)), so the answer is ineligible."
-us,scenario_028,child3_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model said outright that PolicyEngine's CHIP rules do not exclude children with ESI. That is wrong: PolicyEngine applies a disqualifying-coverage test under 42 U.S.C. 1397jj(b)(1)(C), and Child 3's employer coverage fails it. It also used an outdated $32,150 guideline instead of $33,000, though income still passes either way."
-us,scenario_028,child3_chip_eligible,claude-opus-5,llm_error,health_coverage,False,"The model checked only age (under 19) and income (about 190% FPL, within PA's limit). It never applied the rule that a child covered by employer-sponsored group insurance is not a targeted low-income child, which is the test that makes Child 3 ineligible."
-us,scenario_028,child3_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"The model correctly found income above the Medicaid limit and below PA's 314% CHIP limit. It skipped the CHIP requirement that the child have no group health coverage, and Child 3's listed employer-sponsored insurance fails that requirement."
-us,scenario_028,child3_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"The model based eligibility only on income (about 186% FPL, between the Medicaid and CHIP limits). It disregarded Child 3's employer-sponsored insurance, which disqualifies the child under 42 CFR 457.310(b)(2)(ii)."
-us,scenario_028,child3_chip_eligible,deepseek-v4-flash-0731,llm_error,health_coverage,False,"The model treated an income in PA's 158–314% FPL CHIP band as enough for eligibility. It never applied the exclusion for children already covered by a group health plan, and Child 3's employer-sponsored insurance triggers that exclusion."
-us,scenario_028,child3_chip_eligible,deepseek-v4.1-flash,llm_error,health_coverage,False,"The model checked only age and income against a 300% FPL threshold. It ignored that Child 3 has employer-sponsored insurance, which makes the child not a targeted low-income child and so ineligible for CHIP."
-us,scenario_028,child3_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"The model concluded eligibility from age under 19 and income below 314% FPL alone. It omitted CHIP's disqualifying-coverage test, which Child 3 fails because of the employer-sponsored insurance."
-us,scenario_028,child3_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"The model applied only the age and income limits. It did not account for Child 3's employer-sponsored group coverage, which excludes the child from CHIP under 42 U.S.C. 1397jj(b)(1)(C)."
-us,scenario_028,child3_chip_eligible,gemini-3.8-flash,llm_error,health_coverage,False,"The model checked only age and income. It missed the rule that a child with employer-sponsored group health coverage is not a targeted low-income child, which makes Child 3 CHIP-ineligible."
-us,scenario_028,child3_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no value or explanation for child3_chip_eligible, so there was no answer to score. The correct derivation passes age and income (182% FPL against a 319% limit) but fails CHIP because Child 3 has employer-sponsored insurance."
-us,scenario_028,child3_chip_eligible,glm-5.3,llm_error,health_coverage,False,"The model correctly placed income at about 185% FPL, above the children's Medicaid limit and within the CHIP limit. It never applied the disqualifying-coverage test, which Child 3's employer-sponsored insurance fails."
-us,scenario_028,child3_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"The model based eligibility only on age and PA's CHIP income range. It ignored the listed employer-sponsored insurance, which disqualifies Child 3 under 42 CFR 457.310(b)(2)(ii)."
-us,scenario_028,child3_chip_eligible,grok-4.6,llm_error,health_coverage,False,"The model correctly found Child 3 not Medicaid-eligible and income below about 314% FPL. It then treated those two facts as enough and skipped the requirement that the child have no group health coverage, which Child 3's employer-sponsored insurance violates."
-us,scenario_028,child3_chip_eligible,grok-4.7,llm_error,health_coverage,False,"The model concluded eligibility from the age and income rules alone, as its own reasoning says. It omitted CHIP's coverage test: a child with employer-sponsored group insurance is not a targeted low-income child, so Child 3 is ineligible."
-us,scenario_028,child3_chip_eligible,kimi-k3,llm_error,health_coverage,False,"The model got the income test right (about 187% FPL against a 319% limit). It wrongly asserted that current ESI does not negate eligibility, when group health coverage disqualifies a child under 42 U.S.C. 1397jj(b)(1)(C)."
-us,scenario_028,child3_chip_eligible,minimax-m3,llm_error,health_coverage,False,"The model counted the $1,267 of child support as income, giving $61,277 instead of $60,010 MAGI, although income still passes either way. The decisive error is that it ignored Child 3's employer-sponsored insurance, which disqualifies the child from CHIP."
-us,scenario_028,child3_chip_eligible,muse-spark-1.3,llm_error,health_coverage,False,"The model checked only age and income, using a 300% FPL limit. It never applied the exclusion for children covered by a group health plan, and Child 3's employer-sponsored insurance triggers that exclusion."
-us,scenario_028,child3_chip_eligible,ox-alpha,llm_error,health_coverage,False,"The model explicitly said existing insurance coverage does not affect eligibility. That is wrong: PolicyEngine's CHIP rules disqualify a child covered by employer-sponsored group insurance (42 CFR 457.310(b)(2)(ii)), so Child 3 is ineligible."
-us,scenario_028,child3_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model wrongly asserted that ESI does not disqualify a child from CHIP in PA. It also claimed PA CHIP has no income limit, but PolicyEngine applies a 319% FPL cap and excludes children with group health coverage. Child 3's employer-sponsored insurance makes the child ineligible."
-us,scenario_028,child3_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,"The model relied only on income being within PA's CHIP range. It skipped the disqualifying-coverage test, which Child 3 fails because the child has employer-sponsored insurance."
+us,scenario_028,child3_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"The model checked only income (~185% FPL, between the Medicaid limit and the ~314% CHIP limit) and never applied the CHIP rule that excludes a child already covered by a group health plan. Child 3's listed employer-sponsored insurance disqualifies them under 42 CFR 457.310(b)(2)(ii), so the answer is 0."
+us,scenario_028,child3_chip_eligible,claude-fable-5.2,llm_error,health_coverage,False,"The model got the age and income tests right (182% FPL, above Medicaid and below 314%). It then wrongly 'assumed existing employer coverage does not bar eligibility.' Coverage under a group health plan excludes a child from being a targeted low-income child, so Child 3's ESI makes them CHIP-ineligible."
+us,scenario_028,child3_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model claimed that PolicyEngine's CHIP eligibility does not exclude children with ESI. That is wrong: PolicyEngine applies the 42 U.S.C. 1397jj(b)(1)(C) group-health-plan exclusion, and Child 3's employer-sponsored insurance fails it. Its income test (well under 314% FPL) was not the deciding step."
+us,scenario_028,child3_chip_eligible,claude-opus-5,llm_error,health_coverage,False,"The model checked only age (under 19) and income (~190% FPL against PA's CHIP limit). It ignored Child 3's employer-sponsored insurance, which disqualifies them from CHIP under the targeted-low-income-child definition's exclusion for group health plan coverage."
+us,scenario_028,child3_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,"The model correctly placed income (~187% FPL) above the Medicaid limit and below the ~314% CHIP limit, but it never applied the CHIP coverage test. Child 3 has employer-sponsored insurance, and a child covered by a group health plan is not CHIP-eligible."
+us,scenario_028,child3_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"The model stopped at the income test (186% FPL, between the Medicaid and 314% CHIP limits) and skipped the exclusion for children with other creditable group coverage. Child 3's listed employer-sponsored insurance makes them CHIP-ineligible."
+us,scenario_028,child3_chip_eligible,deepseek-v4-flash-0731,llm_error,health_coverage,False,"The model placed household income within an assumed 158–314% FPL CHIP band and concluded Child 3 was eligible. It ignored that Child 3 has employer-sponsored insurance, which by statute excludes them from being a targeted low-income child under CHIP."
+us,scenario_028,child3_chip_eligible,deepseek-v4.1-flash,llm_error,health_coverage,False,"The model applied only an age test and a 300% FPL income cap (PA's actual limit is 314% plus the 5-point disregard). It never applied the group-health-plan exclusion, so it missed that Child 3's employer-sponsored insurance bars CHIP eligibility."
+us,scenario_028,child3_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"The model treated being under 19 with income below 314% FPL as sufficient for CHIP. It omitted the requirement that the child not be covered by a group health plan, and Child 3's employer-sponsored insurance disqualifies them."
+us,scenario_028,child3_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"The model checked only age and the PA income limit and ignored Child 3's listed employer-sponsored insurance. Under 42 CFR 457.310(b)(2)(ii), that coverage makes Child 3 ineligible for CHIP."
+us,scenario_028,child3_chip_eligible,gemini-3.8-flash,llm_error,health_coverage,False,The model checked only age and the PA income limit and never applied CHIP's disqualifying-coverage test. Child 3's employer-sponsored insurance excludes them from being a targeted low-income child.
+us,scenario_028,child3_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable value or explanation for child3_chip_eligible, so there was no substantive answer to score. The correct answer is 0, because Child 3's employer-sponsored insurance disqualifies them from CHIP."
+us,scenario_028,child3_chip_eligible,glm-5.3,llm_error,health_coverage,False,"The model correctly found income (~185% FPL) above the children's Medicaid limit and within PA's CHIP limit. It ignored Child 3's employer-sponsored insurance, which by statute excludes a child covered by a group health plan from CHIP."
+us,scenario_028,child3_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,The model concluded eligibility from age 10 and income within PA's CHIP range alone. It missed the coverage requirement: Child 3's employer-sponsored insurance makes them not a targeted low-income child and therefore CHIP-ineligible.
+us,scenario_028,child3_chip_eligible,grok-4.6,llm_error,health_coverage,False,"The model correctly established that Child 3 is not Medicaid-eligible and that MAGI is below PA's ~314% CHIP limit. It never applied the disqualifying-coverage test, and Child 3's employer-sponsored insurance fails that test, so CHIP eligibility is 0."
+us,scenario_028,child3_chip_eligible,grok-4.7,llm_error,health_coverage,False,"The model explicitly limited its check to 'income and age rules' and omitted CHIP's third condition, that the child not be covered by a group health plan. Child 3's listed employer-sponsored insurance disqualifies them."
+us,scenario_028,child3_chip_eligible,kimi-k3,llm_error,health_coverage,False,"The model correctly placed income (~187% FPL) between the Medicaid limit and the 319% CHIP limit. It then wrongly asserted that 'current ESI coverage does not negate eligibility.' Group health plan coverage excludes a child from CHIP under 42 U.S.C. 1397jj(b)(1)(C), so Child 3 is ineligible."
+us,scenario_028,child3_chip_eligible,minimax-m3,llm_error,health_coverage,False,"The model wrongly counted the $1,267 of child support in household income ($61,277), but child support is not MAGI income. That error did not change the income result. The decisive mistake is that it ignored Child 3's employer-sponsored insurance, which excludes them from CHIP regardless of income."
+us,scenario_028,child3_chip_eligible,muse-spark-1.3,llm_error,health_coverage,False,The model used a 300% FPL limit (PA's is 314% plus the 5-point disregard) and checked only age and income. It missed that Child 3's employer-sponsored insurance disqualifies them from CHIP under the group-health-plan exclusion.
+us,scenario_028,child3_chip_eligible,ox-alpha,llm_error,health_coverage,False,"The model explicitly stated that 'existing insurance coverage ... do[es] not affect eligibility.' That is backwards: a child covered by a group health plan is not a targeted low-income child, so Child 3's employer-sponsored insurance makes them CHIP-ineligible even though income (~185% FPL) passes."
+us,scenario_028,child3_chip_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model asserted that ESI does not disqualify CHIP eligibility in PA and that PA CHIP has no income limit. Both claims are wrong. PolicyEngine applies a 319% FPL CHIP income cap and the group-health-plan exclusion, and Child 3's employer-sponsored insurance makes them ineligible."
+us,scenario_028,child3_chip_eligible,qwen3.8-max,llm_error,health_coverage,False,The model concluded eligibility from the income range alone and never considered coverage. Child 3's employer-sponsored insurance excludes them from CHIP under 42 CFR 457.310(b)(2)(ii).
us,scenario_028,child3_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,child3_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,child3_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly treated a higher CHIP-funded coverage threshold of 215% FPL as a Medicaid threshold. Child3's 1.82-times-FPL MAGI exceeds the applicable Medicaid limits, and CHIP eligibility cannot be substituted for Medicaid eligibility."
@@ -2684,147 +2722,150 @@ us,scenario_028,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,
us,scenario_028,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed TCJA expired, used a 15% bracket and a $1,000-per-child CTC, and even left out the personal exemptions its own expiration premise requires. 2026 law uses the $24,150 HoH deduction and 10%/12% brackets, giving $3,949.20 of tax that the $6,600 CTC fully offsets."
us,scenario_028,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value for federal_income_tax_before_refundable_credits, so there was no answer to score against the $0 reference."
us,scenario_028,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It used a $2,000-per-child CTC instead of $2,200 and subtracted $5,100 of refundable ACTC first, leaving only $900 as nonrefundable. The nonrefundable CTC is applied first up to the full tax liability, which zeroes out the roughly $3,950 tax."
-us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own work reached about $2,916: refundable CTC of $6,600 − $3,935 = $2,665, plus about $251 of EITC. It then submitted $5,100, the 3 × $1,700 refundable cap, which skips the rule that ACTC is limited to the CTC left after the nonrefundable portion offsets tax. Its EITC was also too low because it used a $23,000 phase-out start and a lower maximum instead of $23,890 and $8,231, which give $624.13."
-us,scenario_028,federal_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It started the EITC phase-out at $23,000 instead of the 2026 non-joint threshold of $23,890, which gave $436.70 instead of $624.13. Its pre-credit tax of about $3,955 was also slightly off from the correct $3,949.20, so its refundable CTC came out at $2,644.80 instead of $2,650.80."
-us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It counted nontaxable child support in income and declared the household ineligible for EITC, but the 3-child EITC is still $624.13 at $60,010 AGI. Its $3,850 refundable CTC figure does not follow from any rule. The correct refundable CTC is the $2,650.80 of the $6,600 credit left after offsetting $3,949.20 of tax."
-us,scenario_028,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"It claimed tax liability fully absorbs the CTC, but pre-credit tax is only $3,949.20 against a $6,600 CTC, so $2,650.80 is refundable. It also wrongly found no EITC, when the 3-child EITC at $60,010 is $624.13."
-us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000-per-child CTC ($6,000) instead of $2,200 per child ($6,600), which gave $2,037 of unused CTC instead of $2,650.80. It also said the 3-child EITC ends at about $59,899, but the 2026 phase-out runs to about $62,970, which leaves $624.13 at $60,010."
-us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $6,000 CTC and an inflated pre-credit tax of about $4,800, then submitted $4,500, which contradicts its own estimate of about $1,200 of ACTC. The correct figures are a $6,600 CTC and $3,949.20 of tax, leaving $2,650.80 refundable, plus $624.13 of EITC, which it wrongly zeroed."
-us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,other,False,"It awarded the full $1,700-per-child refundable cap ($5,100) and never applied the rule that ACTC cannot exceed the CTC left after the nonrefundable portion offsets the $3,949.20 tax, which limits it to $2,650.80. It also wrongly zeroed the EITC, which is $624.13."
-us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $2,000-per-child CTC, a $22,500 standard deduction and 2024 brackets, which gave $4,170 of tax and $1,830 of ACTC. The correct inputs are $2,200 per child, $24,150 and 2026 brackets, giving $3,949.20 of tax and $2,650.80 of ACTC. It also wrongly put the 3-child EITC phase-out end near $58,000; the 2026 EITC at $60,010 is still $624.13."
-us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It assumed a $2,000-per-child CTC and an EITC phase-out end of about $56,838, then guessed $2,000 total. The correct total is $2,650.80 of unused CTC ($6,600 − $3,949.20) plus $624.13 of EITC."
-us,scenario_028,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $6,000 CTC and $4,170 of tax, giving $1,830 of ACTC instead of $6,600 − $3,949.20 = $2,650.80. Its EITC of $165 also understates the $624.13 that the $8,231 maximum, $23,890 threshold and 21.06% rate produce."
-us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It claimed tax liability exceeds the CTC, but pre-credit tax of $3,949.20 is well below the $6,600 CTC, leaving $2,650.80 refundable. It also wrongly treated the EITC as fully phased out at $60,000, when it is still $624.13."
-us,scenario_028,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,other,False,"Its EITC estimate of $662.87 was close, but it dropped the refundable CTC entirely by assuming the nonrefundable CTC is fully used against tax. In fact $6,600 − $3,949.20 = $2,650.80 of CTC is left and refundable."
-us,scenario_028,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $6,000 CTC and $4,073.20 of tax, giving $1,926.80 of ACTC instead of $6,600 − $3,949.20 = $2,650.80. Its EITC also used a $8,271 maximum and $24,024 threshold instead of $8,231 and $23,890, which overstated it at $692.35 versus $624.13."
-us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Its $66 of leftover CTC is consistent with a sunset-era CTC sized roughly to cover the tax. The OBBBA $6,600 CTC exceeds the $3,949.20 tax by $2,650.80, all of it refundable. Its EITC of $524 also falls short of the $624.13 that the 2026 parameters give."
-us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It reported no refundable credits, missing both the $2,650.80 of refundable CTC (the $6,600 CTC minus $3,949.20 of tax) and the $624.13 EITC still available to a 3-child filer at $60,010."
-us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,other,False,"It estimated the EITC at $574 instead of $624.13, and it wrongly said the CTC was fully used against tax. The $6,600 CTC exceeds the $3,949.20 tax by $2,650.80, which is refundable."
-us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a $3,000 CTC ($1,000 per child, as if TCJA had expired) and $3,039 of tax, so it found no ACTC. OBBBA sets the 2026 CTC at $2,200 per child ($6,600), which leaves $2,650.80 refundable. It also wrongly found no EITC, when $624.13 remains at $60,010."
-us,scenario_028,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It zeroed all refundable credits. In fact the $6,600 CTC exceeds the $3,949.20 tax, leaving $2,650.80 of refundable CTC, and the 3-child EITC at $60,010 is still $624.13."
-us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $6,000 CTC and $4,089.20 of tax, giving $1,910.80 instead of $6,600 − $3,949.20 = $2,650.80. It also left out the $624.13 EITC entirely."
-us,scenario_028,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It claimed income exceeds the EITC phase-out and the CTC is fully absorbed by tax, but the 3-child EITC is still $624.13 at $60,010. Only $3,949.20 of the $6,600 CTC is absorbed, leaving $2,650.80 refundable."
-us,scenario_028,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its ACTC of $1,838.80 implies a $6,000 CTC and about $4,161 of tax, instead of $6,600 and $3,949.20, which leave $2,650.80. Its EITC of $327.51 also understates the $624.13 from the $8,231 maximum and $23,890 threshold."
-us,scenario_028,federal_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"The EITC ($624) and the $6,600 CTC were right, but it understated pre-credit tax at $3,837.60. The correct tax is $3,949.20: taxable income of $60,010 − $24,150 = $35,860 run through the 2026 head-of-household brackets. That error overstated the refundable CTC by $111.60."
-us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits. The correct derivation is $2,650.80 of refundable CTC plus $624.13 of EITC, for $3,274.93."
-us,scenario_028,federal_refundable_credits,glm-5.3,llm_error,other,False,"It used an $1,800-per-child refundable cap instead of $1,700, and paid the cap in full without limiting ACTC to the CTC left after offsetting the $3,949.20 tax ($2,650.80). It also wrongly zeroed the $624.13 EITC."
-us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It treated a full $2,000 per child as refundable. The refundable portion is capped at $1,700 per child and limited to the CTC left after offsetting the $3,949.20 tax, which yields $2,650.80. It also left out the $624.13 EITC."
-us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It reported no refundable credits, missing the $2,650.80 of refundable CTC (the $6,600 CTC minus $3,949.20 of tax) and the $624.13 EITC that remains for 3 children at $60,010."
-us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It applied only $2,184.20 of nonrefundable CTC, far below the correct pre-credit tax of $3,949.20 on $35,860 of taxable income, so its ACTC was inflated. It also wrongly zeroed the $624.13 EITC."
-us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The refundable CTC of $2,650.80 was right, but it wrongly declared the EITC unavailable. With three children the phase-out runs past $60,010, leaving $8,231 − 21.06% × ($60,010 − $23,890) = $624.13."
-us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,other,False,"It paid the full $1,700-per-child refundable cap ($5,100) without limiting ACTC to the CTC left after the nonrefundable portion offsets the $3,949.20 tax, which gives $2,650.80. It also left out the $624.13 EITC."
-us,scenario_028,federal_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"Its refundable CTC of $2,651 was right, but it rounded the EITC to about $650 instead of computing $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. That overstated the total by about $26."
-us,scenario_028,federal_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It computed the refundable CTC correctly ($6,600 − $3,949.20 = $2,650.80) but left out the EITC. For three children at $60,010 AGI the EITC is still $8,231 − 21.06% × ($60,010 − $23,890) = $624.13."
-us,scenario_028,federal_refundable_credits,gpt-6.1-terra,llm_error,other,False,"It gave no derivation, and its $3,430 overstates the correct sum of $2,650.80 refundable CTC (the $6,600 CTC minus $3,949.20 of tax) and $624.13 EITC by about $155. That pattern fits an overestimated EITC or understated pre-credit tax."
-us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It claimed no refundable credits qualify, missing both the $624.13 EITC still available for 3 children at $60,010 and the $2,650.80 of CTC left after offsetting the $3,949.20 tax."
-us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It left only about $40 of ACTC, which implies a CTC roughly equal to the tax. The OBBBA $2,200-per-child CTC ($6,600) exceeds the $3,949.20 tax by $2,650.80. Its EITC of about $660 also overstates the $624.13 from the $8,231 maximum and $23,890 threshold."
-us,scenario_028,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It included about $48 of ACTC, understating the correct $2,650.80 by about $2,602.80. Its EITC of $636 came from an $8,239 maximum and $23,910 threshold instead of $8,231 and $23,890 ($624.13)."
-us,scenario_028,federal_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its tax of $3,948 was essentially right, but its $2,952 of leftover CTC implies a $6,900 CTC ($2,300 per child) instead of $6,600, which overstated ACTC by about $300. Its EITC of $661 also exceeds the correct $624.13."
-us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a sunset-era $3,000 CTC ($1,000 per child), so it found no refundable CTC. OBBBA's $6,600 CTC leaves $2,650.80 refundable after the $3,949.20 tax. Its EITC used an $8,302 maximum and $24,099 threshold, giving $739 instead of $624.13."
-us,scenario_028,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The refundable CTC of about $2,649 was correct, but it estimated the EITC at about $340 instead of $8,231 − 21.06% × ($60,010 − $23,890) = $624.13, which understated the total."
-us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits. The correct derivation is $2,650.80 of refundable CTC plus $624.13 of EITC, for $3,274.93."
-us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The EITC of $624.13 was exactly right, but it put pre-credit tax at $3,838.10 instead of $3,949.20. The correct figure comes from $60,010 − $24,150 = $35,860 of taxable income in the 2026 head-of-household brackets, so its refundable CTC was $111.10 too high ($2,761.90 vs $2,650.80)."
-us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,other,False,"It set ACTC to the $5,100 cap for three children without limiting it to the CTC left after the nonrefundable portion offsets the $3,949.20 tax, which limits it to $2,650.80. It also zeroed the EITC even though its own ~$61,500 phase-out end is above $60,010; the correct EITC is $624.13."
-us,scenario_028,federal_refundable_credits,muse-spark-1.3,llm_error,credit_phaseout,False,"It put the EITC at about $1,500, far above the $624.13 that the 21.06% phase-out from $23,890 leaves at $60,010. Its refundable CTC of about $2,592 was also slightly below the correct $2,650.80."
-us,scenario_028,federal_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"The refundable CTC was about right at $2,650, but it rounded the EITC to about $650 instead of computing $624.13 from the $8,231 maximum, $23,890 threshold and 21.06% rate. It then rounded the total to $3,300."
-us,scenario_028,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The refundable CTC of $2,650.80 was right, but it started the EITC phase-out at $22,210 instead of the 2026 non-joint threshold of $23,890, and used an $8,234 maximum instead of $8,231. That understated the EITC at $273.32 instead of $624.13."
-us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $6,000 CTC and $4,081.20 of tax, giving $1,918.80 of ACTC instead of $6,600 − $3,949.20 = $2,650.80. Its EITC came from improvised inflation of older parameters ($8,222 maximum, $25,914 threshold) and reached $1,042 instead of $624.13."
-us,scenario_028,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"It said the 15%-of-earnings-over-$2,500 limit binds, but that limit is $8,625 and does not bind. The binding limit is the CTC left after offsetting the $3,949.20 tax ($2,650.80). It also left out the $624.13 EITC."
-us,scenario_028,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"The model correctly found that income is above the 130% FPL free-meal limit, but it treated that federal test as the only route to free meals. It never applied Pennsylvania's universal free breakfast under Act 33, which makes breakfast free for all three children regardless of income and creates positive free meal support."
-us,scenario_028,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model stopped once income was above 130% FPL with no SNAP or TANF categorical eligibility. It ignored Pennsylvania's universal free school breakfast (24 P.S. § 13-1337.1(c.2)), which gives free breakfast regardless of income and makes free meal support positive."
-us,scenario_028,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model counted a household of 5 instead of 4 and left out the $10 of interest. Its main mistake was relying only on the federal 130% FPL free-meal test and missing Pennsylvania's universal free breakfast, which makes breakfast free for the three school-age children at any income."
-us,scenario_028,free_school_meals_eligible,claude-haiku-5.5,llm_error,state_local_rule,False,"The model applied only the federal income limit for free meals. It missed that Pennsylvania's Act 33 free breakfast program gives every student at participating schools free breakfast regardless of household income, which produces positive free meal support."
-us,scenario_028,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model correctly put income above the 185% reduced-price limit; it is about 186% of the guideline. More importantly, it treated federal income tiers as the whole test and missed Pennsylvania's universal free breakfast, which makes breakfast free regardless of income."
-us,scenario_028,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model correctly said income exceeds the 185% limit; it is about 186% of the guideline. It also relied only on the federal 130% FPL free-meal threshold and never applied Pennsylvania's universal free breakfast under Act 33, which provides free breakfast to all three children regardless of income."
-us,scenario_028,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model compared income only against the federal 130% FPL free limit. It missed Pennsylvania's income-blind universal free school breakfast, which makes free meal support positive for the three K-12 children."
-us,scenario_028,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"The model correctly totaled $61,277 of income, but it applied only the federal 130% FPL free-meal test. It missed Pennsylvania's Act 33 universal free breakfast, which provides free breakfast regardless of income."
-us,scenario_028,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model's 130% FPL figure was wrong ($47,645), but the decisive error was treating the federal free-meal income test as the only route. Pennsylvania's universal free breakfast gives the children free breakfast regardless of income, so free meal support is positive."
-us,scenario_028,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"The model counted a family of 5 instead of 4 and relied only on the federal 130% FPL free threshold. It missed Pennsylvania's universal free school breakfast, which makes breakfast free for all three children at any income."
-us,scenario_028,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model stopped at income above the federal 130% FPL free-meal limit. It never applied Pennsylvania's Act 33 universal free breakfast, which provides free breakfast regardless of income and creates positive free meal support."
-us,scenario_028,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model applied only the federal 130% FPL free-meal test, and it also left out the $10 of interest. It missed Pennsylvania's universal free breakfast for students at participating schools, which is granted regardless of income."
-us,scenario_028,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model left the $1,267 of child support out of countable income and used only the federal 130% FPL free-meal limit. It missed Pennsylvania's income-blind universal free breakfast, which makes free meal support positive."
-us,scenario_028,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model correctly found income of $61,277 above 130% FPL, but it treated that federal test as decisive. It missed Pennsylvania's Act 33 universal free breakfast, which gives the three children free breakfast regardless of income."
-us,scenario_028,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model compared income only to the federal 130% FPL free threshold, and it left out the $10 of interest. It never applied Pennsylvania's universal free school breakfast, which is free regardless of income."
-us,scenario_028,free_school_meals_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model applied only the federal 130% FPL free-meal threshold. It missed Pennsylvania's universal free breakfast under 24 P.S. § 13-1337.1(c.2), which makes breakfast free for the three K-12 children at any income."
-us,scenario_028,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model concluded no free meals from the federal income threshold alone. It missed Pennsylvania's income-blind universal free breakfast, which yields positive free meal support for the three children."
-us,scenario_028,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model correctly placed income above 130% FPL but treated the federal free-meal test as the only route. It missed Pennsylvania's Act 33 universal free school breakfast, which applies regardless of household income."
-us,scenario_028,free_school_meals_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,"The model relied only on the federal 130% FPL limit. It overlooked Pennsylvania's universal free breakfast program, which gives free breakfast to students at participating schools regardless of income and produces positive free meal support."
-us,scenario_028,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model applied only the federal 130% poverty-guideline free-meal test. It missed Pennsylvania's income-blind universal free school breakfast, which makes the household's free meal support positive."
-us,scenario_028,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"The model used only the federal 130% FPL free-meal limit, with an understated figure of about $40,560. It missed Pennsylvania's universal free breakfast, which provides free breakfast regardless of income."
-us,scenario_028,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"The model stopped at the federal 130% FPL threshold. It never applied Pennsylvania's Act 33 universal free school breakfast, which makes breakfast free for all three K-12 children regardless of income."
-us,scenario_028,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"The model treated the federal 130% poverty income test as the whole determination. It missed Pennsylvania's universal free breakfast, which gives free breakfast at any income and makes free meal support positive."
-us,scenario_028,free_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,"The model applied only the federal 130% FPL free-meal test for a family of 4. It missed Pennsylvania's income-blind universal free school breakfast, which puts breakfast in the free tier for the three children."
-us,scenario_028,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for free_school_meals_eligible, so there was no answer to score."
-us,scenario_028,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model correctly found income above 130% of poverty but treated that federal test as decisive. It missed Pennsylvania's universal free breakfast under Act 33, which provides free breakfast regardless of income."
-us,scenario_028,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,"The model applied only the federal income threshold for free meals. It ignored Pennsylvania's universal free school breakfast, which gives the three children free breakfast regardless of income and produces positive free meal support."
-us,scenario_028,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,The model looked only for SNAP/TANF categorical eligibility or very low income. It missed that Pennsylvania's Act 33 universal free breakfast gives every student at participating schools free breakfast regardless of income.
-us,scenario_028,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model checked only the federal free-meal income limit and direct certification. It missed Pennsylvania's universal free school breakfast, which PolicyEngine applies regardless of income and values at $1,112.40 a year for the three children."
-us,scenario_028,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model treated the federal free-meal income threshold as the only route. It missed Pennsylvania's income-blind universal free breakfast, which makes free meal support positive."
-us,scenario_028,free_school_meals_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,"The model applied the federal 130% FPL test and direct certification only. It never applied Pennsylvania's universal free breakfast under 24 P.S. § 13-1337.1(c.2), which provides free breakfast regardless of income."
-us,scenario_028,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model relied only on the federal free-meal income threshold. It missed Pennsylvania's universal free school breakfast, which puts breakfast in the free tier for all three children at any income."
-us,scenario_028,free_school_meals_eligible,gpt-6-astra,llm_error,state_local_rule,False,"The model correctly found income above 130% of poverty and no categorical eligibility, but it never applied Pennsylvania's Act 33 universal free breakfast. That program grants free breakfast regardless of income and makes free meal support positive."
-us,scenario_028,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"The model applied only the federal free-meal income limit and categorical eligibility. It missed Pennsylvania's universal free school breakfast, which is income-blind and yields positive free meal support."
-us,scenario_028,free_school_meals_eligible,gpt-6-sol,llm_error,state_local_rule,False,"The model stopped at the federal free-meal threshold and the absence of categorical eligibility. It overlooked Pennsylvania's universal free breakfast, which gives the three K-12 children free breakfast regardless of income."
-us,scenario_028,free_school_meals_eligible,gpt-6.1-sol,llm_error,state_local_rule,False,"The model correctly totaled $61,277 above the four-person free-meal limit, but it treated the federal test as the only route. It missed Pennsylvania's Act 33 universal free breakfast, which makes breakfast free regardless of income."
-us,scenario_028,free_school_meals_eligible,gpt-6.1-terra,llm_error,state_local_rule,False,"The model applied only the federal free-meal income threshold. It missed Pennsylvania's income-blind universal free school breakfast, which produces positive free meal support for the three children."
-us,scenario_028,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,"The model applied only an income threshold for free meals. It missed Pennsylvania's universal free breakfast, which gives free breakfast to students at participating schools regardless of household income."
-us,scenario_028,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model correctly found income of $61,277 above 130% FPL, but it treated the federal test as decisive. It missed Pennsylvania's Act 33 universal free breakfast, which is provided regardless of income."
-us,scenario_028,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"The model checked the 130% FPL test and categorical eligibility only. It missed Pennsylvania's universal free school breakfast, which PolicyEngine applies regardless of income, making free meal support positive."
-us,scenario_028,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"The model relied only on the federal 130% FPL free-meal cutoff. It never applied Pennsylvania's income-blind universal free breakfast, which puts breakfast in the free tier for the three children."
-us,scenario_028,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model correctly placed income above the federal 130% FPL free threshold. It missed Pennsylvania's universal free breakfast under 24 P.S. § 13-1337.1(c.2), which gives free breakfast regardless of income."
-us,scenario_028,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,"The model applied only the federal 130% FPL free-meal limit. It missed Pennsylvania's Act 33 universal free school breakfast, which makes breakfast free for all three children at any income."
-us,scenario_028,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"The model checked only the 130% FPL income test and categorical program receipt. It missed Pennsylvania's universal free breakfast, a separate state route that gives free breakfast regardless of income."
-us,scenario_028,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model overstated the income ratio at 191% (the engine's ratio is 1.86) and treated the federal 130% cutoff and SNAP/TANF categorical eligibility as the only routes. It missed Pennsylvania's income-blind universal free school breakfast, which makes free meal support positive."
-us,scenario_028,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,"The model applied only the federal 130% FPL free-meal limit, with an understated figure of about $40,500. It never applied Pennsylvania's universal free breakfast, which provides free breakfast regardless of income."
-us,scenario_028,free_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,"The model stopped at income above 130% FPL with no categorical eligibility. It missed Pennsylvania's Act 33 universal free breakfast, which gives the three children free breakfast regardless of income."
-us,scenario_028,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"The model correctly counted child support and found income above 130% of poverty, but it treated that federal test as decisive. It missed Pennsylvania's universal free school breakfast, which is income-blind and makes free meal support positive."
-us,scenario_028,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model left the $1,267 of child support out of income and used only the federal 130% FPL free-meal threshold. It missed Pennsylvania's universal free breakfast, which makes breakfast free for the three children regardless of income."
-us,scenario_028,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"The model correctly totaled $61,277 but applied only the federal 130% FPL free-meal limit. It missed Pennsylvania's Act 33 universal free school breakfast, which gives free breakfast regardless of income."
-us,scenario_028,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,"The model applied only the federal free-meal income limit. It overlooked Pennsylvania's income-blind universal free breakfast, which produces positive free meal support for the three K-12 children."
+us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model computed refundable CTC as unused credit (~$2,665) plus EITC (~$251) ≈ $2,916, then submitted the $5,100 per-child refundable cap instead. That cap does not bind here, because refundable CTC is limited to the $2,650.80 of $6,600 left after $3,949.20 of tax. Its EITC also used an $8,046 maximum and a $23,000 phaseout start instead of $8,231 and $23,890, which give $624.13."
+us,scenario_028,federal_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"The model started the 3-child EITC phaseout at $23,000 instead of the 2026 HoH threshold of $23,890, which cut the EITC to $436.70 instead of $624.13. It also overstated pre-credit tax ($3,955 vs $3,949.20), so its refundable CTC came out at $2,644.80 instead of $2,650.80."
+us,scenario_028,federal_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"The EITC ($624.13) is correct, but pre-credit tax of $3,831.60 is $117.60 too low. That equals 12% × $980, so the model deducted the $980 of auto loan interest as qualified vehicle-loan interest. PolicyEngine does not allow that deduction for this household, so pre-credit tax is $3,949.20 and refundable CTC is $2,650.80, not $2,768.40."
+us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model used a $2,000 per-child CTC instead of $2,200 and made up a $3,850 refundable amount. It never limited refundable CTC to the credit left after offsetting $3,949.20 of tax, which gives $2,650.80. It also wrongly counted child support in income and set EITC to zero, even though the 3-child EITC at $60,010 is $624.13."
+us,scenario_028,federal_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"The model wrongly said the CTC is fully absorbed by tax. The $6,600 CTC is larger than the $3,949.20 pre-credit tax, which leaves $2,650.80 refundable. It also missed that the 3-child EITC is still $624.13 at $60,010, because it phases out only near $62,973."
+us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the pre-OBBBA $2,000 per-child CTC ($6,000) and a $23,000 standard deduction, which left $2,037 of unused CTC. The correct figures are $6,600 of CTC and $3,949.20 of tax, leaving $2,650.80. It also took the 3-child EITC as fully phased out at $59,899, but the 2026 parameters ($8,231 max, $23,890 start) still give $624.13."
+us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model's own work found ~$1,200 of refundable CTC from a $6,000 CTC, but it submitted an unsupported $4,500. It used $2,000 per child instead of $2,200, overstated pre-credit tax (~$4,800 vs $3,949.20), and wrongly set EITC to zero instead of $624.13."
+us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model used the $1,700 × 3 = $5,100 refundable cap as the refundable CTC. It skipped the rule that refundable CTC is limited to the CTC not used against tax: $6,600 − $3,949.20 = $2,650.80. It also wrongly said there is no EITC, when the 3-child EITC at $60,010 is $624.13."
+us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $2,000 per-child CTC, a $22,500 HoH standard deduction and a $16,550 10% bracket, which gave $4,170 of tax and $1,830 of unused CTC. The 2026 figures ($2,200 per child, $24,150 deduction, $17,700 bracket) give $2,650.80. It also used a 2025 EITC phaseout end of ~$57,310 to set EITC to zero, but the 2026 3-child EITC at $60,010 is $624.13."
+us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model gave an arbitrary $2,000 without calculating the unused CTC. With $6,600 of CTC and $3,949.20 of tax, refundable CTC is $2,650.80. It also wrongly said the EITC phases out by ~$56,838, when the 2026 3-child EITC at $60,010 is $624.13."
+us,scenario_028,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model started from a $6,000 CTC ($2,000 per child) and $4,170 of tax, which gave $1,830 refundable. The OBBBA $2,200 per-child credit and $3,949.20 of tax give $2,650.80. Its ~$165 EITC is also too low; $8,231 − 21.06% × ($60,010 − $23,890) = $624.13."
+us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model wrongly said tax exceeds the CTC. Pre-credit tax is $3,949.20 against a $6,600 CTC, leaving $2,650.80 refundable. It also wrongly treated the EITC as phased out at $60,000, when the 3-child EITC is still $624.13."
+us,scenario_028,federal_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"The model wrongly said the CTC is fully used against tax, which drops the $2,650.80 refundable CTC left from $6,600 after $3,949.20 of tax. Its EITC of $662.87 is also slightly high compared with $624.13 from the $8,231 maximum and $23,890 phaseout start."
+us,scenario_028,federal_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model used a $6,000 total CTC ($2,000 per child) and $4,073.20 of tax instead of $6,600 and $3,949.20, which understated refundable CTC at $1,926.80 instead of $2,650.80. Its EITC parameters ($8,271 max, $24,024 start) overstated the EITC at $692.35 instead of $624.13."
+us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model left only $66 of unused CTC, which far overstates tax against the CTC. With $6,600 of CTC and $3,949.20 of tax, $2,650.80 is refundable. Its EITC of $524 is also too low compared with $624.13."
+us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model said no refundable credits apply. It missed the $2,650.80 refundable CTC left from $6,600 after $3,949.20 of tax, and the $624.13 3-child EITC still payable at $60,010."
+us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model wrongly said the CTC was fully used against tax. Pre-credit tax of $3,949.20 leaves $2,650.80 of the $6,600 CTC refundable. Its $574 EITC is also slightly low compared with $624.13."
+us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $1,000 per-child CTC ($3,000), as if the TCJA had expired, and a $3,039 tax figure. Under OBBBA the 2026 CTC is $2,200 per child ($6,600), and tax is $3,949.20, leaving $2,650.80 refundable. It also wrongly set EITC to zero instead of $624.13."
+us,scenario_028,federal_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"The model said income phaseouts and tax liability remove all refundable credits. It missed the $2,650.80 refundable CTC (from $6,600 of CTC minus $3,949.20 of tax) and the $624.13 EITC still payable at $60,010 with three children."
+us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $6,000 CTC ($2,000 per child) and $4,089.20 of tax, which gave $1,910.80 refundable instead of $2,650.80 from $6,600 − $3,949.20. It also left out the $624.13 3-child EITC."
+us,scenario_028,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model wrongly said income is above the EITC phaseout and the CTC is fully absorbed. The 3-child EITC at $60,010 is $624.13, and $2,650.80 of the $6,600 CTC is left after $3,949.20 of tax and is refundable."
+us,scenario_028,federal_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model's $1,838.80 refundable CTC comes from a $2,000 per-child ($6,000) credit, but the 2026 CTC is $2,200 per child ($6,600), which gives $2,650.80 after $3,949.20 of tax. Its EITC of $327.51 is also understated compared with $624.13."
+us,scenario_028,federal_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"The model's EITC ($624) is correct, but it put pre-credit tax at $3,837.60 instead of $1,770 + 12% × ($35,860 − $17,700) = $3,949.20. That inflated the refundable CTC to $2,762.40 instead of $2,650.80."
+us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no usable value for federal_refundable_credits. The correct value is $2,650.80 of refundable CTC plus $624.13 of EITC."
+us,scenario_028,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model used an $1,800 per-child refundable cap ($5,400) as the refundable CTC. It skipped the limit to the CTC not used against tax ($6,600 − $3,949.20 = $2,650.80), and the 2026 cap is $1,700 per child. It also wrongly set EITC to zero instead of $624.13."
+us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated the full $2,000 per child as refundable, ignoring both the $1,700 per-child refundable cap and the rule that only CTC left after tax is refunded. The correct refundable CTC is $6,600 − $3,949.20 = $2,650.80, and it also left out the $624.13 EITC."
+us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model said no refundable credits apply. It missed the $2,650.80 refundable CTC left after $3,949.20 of tax is offset, and the $624.13 3-child EITC still payable at $60,010."
+us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model put nonrefundable CTC used against tax at only $2,184.20, far below the actual $3,949.20 pre-credit tax. That inflated refundable CTC to ~$4,416 (submitted as $4,444) instead of $2,650.80. It also wrongly set EITC to zero instead of $624.13."
+us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model's refundable CTC of $2,650.80 is correct, but it wrongly said EITC is unavailable at this income. The 2026 3-child EITC phases out only near $62,973 and is $624.13 at $60,010."
+us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model gave the full $1,700 × 3 = $5,100 refundable cap. It skipped the limit to the CTC left after tax ($6,600 − $3,949.20 = $2,650.80) and left out the $624.13 EITC."
+us,scenario_028,federal_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"The model's refundable CTC (~$2,651) is right, but it rounded the EITC to ~$650 instead of computing $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. That put the total about $26 too high."
+us,scenario_028,federal_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"The model correctly got $2,650.80 of refundable CTC from $6,600 − $3,949.20 but left out the EITC. The 3-child EITC is still in its phaseout range at $60,010 and adds $624.13."
+us,scenario_028,federal_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,"The model gave a rough $3,430 without working it out, overstating the total by $155. The correct parts are $2,650.80 of refundable CTC ($6,600 − $3,949.20) and $624.13 of EITC ($8,231 − 21.06% × $36,120)."
+us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model said no EITC or other refundable credit applies. It missed the $624.13 3-child EITC still payable at $60,010 and the $2,650.80 of the $6,600 CTC left after $3,949.20 of tax."
+us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model left only ~$40 of unused CTC, which far overstates tax against the CTC. With $6,600 of CTC and $3,949.20 of tax, $2,650.80 is refundable. Its ~$660 EITC is also slightly high compared with $624.13."
+us,scenario_028,federal_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"The model put unused CTC at about $48, but $3,949.20 of tax against a $6,600 CTC leaves $2,650.80 refundable. Its $636 EITC (from a $23,910 start and $8,239 max) is also slightly off from $624.13."
+us,scenario_028,federal_refundable_credits,grok-4.7,llm_error,other,False,"The model correctly offset $3,948 of tax but then made a subtraction error, putting the remainder at $2,952 when $6,600 − $3,948 ≈ $2,652. Its $661 EITC is also high compared with $624.13 from the $8,231 maximum and $23,890 phaseout start."
+us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed the CTC reverted to $1,000 per child ($3,000), leaving nothing refundable. Under OBBBA the 2026 CTC is $2,200 per child, which leaves $2,650.80 refundable after $3,949.20 of tax. It also used reverted EITC parameters ($8,302 max, $24,099 start), giving $739 instead of $624.13."
+us,scenario_028,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model's refundable CTC of ~$2,649 is close to the correct $2,650.80. It understated the EITC at ~$340, though applying the $8,231 maximum and $23,890 HoH phaseout start at 21.06% gives $624.13."
+us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no usable value for federal_refundable_credits. The correct value is $2,650.80 of refundable CTC plus $624.13 of EITC."
+us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model's EITC of $624.13 is correct, but it put nonrefundable CTC used against tax at $3,838.10 instead of $3,949.20 ($1,770 + 12% × ($35,860 − $17,700)). That inflated refundable CTC to $2,761.90 instead of $2,650.80."
+us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model set refundable CTC to the $5,100 cap without limiting it to the CTC left after tax ($6,600 − $3,949.20 = $2,650.80). It also wrongly treated the EITC as fully phased out, when it is $624.13 at $60,010."
+us,scenario_028,federal_refundable_credits,muse-spark-1.3,llm_error,credit_phaseout,False,"The model overstated the EITC at ~$1,500. Applying 21.06% to the $36,120 above the $23,890 phaseout start leaves only $624.13 of the $8,231 maximum. Its ~$2,592 refundable CTC is also slightly below $2,650.80."
+us,scenario_028,federal_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"The model rounded both parts (~$2,650 CTC, ~$650 EITC) instead of computing them exactly. The correct EITC is $8,231 − 21.06% × ($60,010 − $23,890) = $624.13, and the correct refundable CTC is $2,650.80."
+us,scenario_028,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model's refundable CTC of $2,650.80 is correct, but it started the EITC phaseout at $22,210 instead of the 2026 single/HoH threshold of $23,890. That cut the EITC to $273.32 instead of $624.13."
+us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a $2,000 per-child CTC ($6,000) and $4,081.20 of tax, which gave $1,918.80 refundable instead of $2,650.80 from $6,600 − $3,949.20. It also inflated the EITC to $1,042 using guessed 5% inflation factors instead of the 2026 $8,231 maximum and $23,890 start, which give $624.13."
+us,scenario_028,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model said the earned-income limit sets refundable CTC at $3,200, but 15% × ($60,000 − $2,500) = $8,625 does not bind. The binding limit is the CTC left after tax, $6,600 − $3,949.20 = $2,650.80, and the $624.13 EITC must be added."
+us,scenario_028,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"It used the federal 130% FPL free-meal test as the only route, so it never applied Pennsylvania's universal free breakfast (Act 33 of 2023). That rule makes breakfast free for all three children regardless of income and creates $1,112.40 of free meal support."
+us,scenario_028,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"It checked only the 130% FPL limit and SNAP/TANF categorical eligibility and missed Pennsylvania's universal free school breakfast. That program gives the three children free breakfast regardless of income, so free meal support is positive."
+us,scenario_028,free_school_meals_eligible,claude-fable-5.2,llm_error,state_local_rule,False,"It stopped at the 130% FPL free-meal limit and SNAP/TANF receipt and never applied Pennsylvania's Act 33 universal free breakfast, which puts breakfast in the free tier for every child. Income is actually 1.82 times FPL (reduced-price lunch), and the free breakfast alone makes the output 1."
+us,scenario_028,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"It treated the household as five people when it has four (one adult and three children). It also applied only the federal 130% FPL free-meal test and missed Pennsylvania's universal free school breakfast, which makes breakfast free for all three children regardless of income."
+us,scenario_028,free_school_meals_eligible,claude-haiku-5.5,llm_error,state_local_rule,False,"It concluded that $60,000 of income rules out free meals, but it never applied Pennsylvania's universal free school breakfast (Act 33 of 2023). That program gives each child a free breakfast regardless of income and produces positive free meal support."
+us,scenario_028,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"It compared wages alone with the 130% and 185% FPL cutoffs and wrongly put the household above the reduced-price limit; its income is actually 1.82 times FPL. It also missed Pennsylvania's universal free breakfast, which gives all three children a free breakfast regardless of income."
+us,scenario_028,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"It applied only the federal 130% FPL free-meal test and wrongly said income also exceeds the 185% reduced-price limit, when income is 1.82 times FPL. It never applied Pennsylvania's Act 33 universal free breakfast, which makes breakfast free for the three children regardless of income."
+us,scenario_028,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"It treated the 130% FPL limit as the only route to free meals and missed Pennsylvania's universal free school breakfast. That program gives the children a free breakfast regardless of income, so free meal support is positive."
+us,scenario_028,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"It counted income correctly at $61,277 but applied only the 130% FPL free-meal limit. It never applied Pennsylvania's universal free breakfast, which gives the three children free breakfast worth $1,112.40 a year regardless of income."
+us,scenario_028,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"It applied only the federal 130% FPL free-meal test, using an overstated cutoff of about $47,645. It missed Pennsylvania's Act 33 universal free breakfast, which puts breakfast in the free tier for all three children regardless of income."
+us,scenario_028,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"It counted the household as five people when it has four, and it applied only the federal 130% FPL free-meal limit. It missed Pennsylvania's universal free school breakfast, which gives each child a free breakfast regardless of income and makes free meal support positive."
+us,scenario_028,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"It applied only the federal 130% FPL free-meal limit. It never applied Pennsylvania's universal free breakfast (Act 33 of 2023), which gives the three children free breakfast regardless of income."
+us,scenario_028,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It tested income only against the 130% FPL free-meal limit and missed Pennsylvania's universal free school breakfast. That program gives all three children a free breakfast regardless of income, so free meal support is positive."
+us,scenario_028,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"It left the $1,267 of child support out of income and applied only the 130% FPL free-meal limit. It never applied Pennsylvania's universal free breakfast, which makes breakfast free for all three children regardless of income."
+us,scenario_028,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It used the 130% FPL limit as the only route to free meals and missed Pennsylvania's Act 33 universal free school breakfast. That program gives the children free breakfast worth $1,112.40 a year regardless of income."
+us,scenario_028,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,It applied only the federal 130% FPL free-meal test and never applied Pennsylvania's universal free school breakfast. That program gives each of the three children a free breakfast regardless of household income.
+us,scenario_028,free_school_meals_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"It compared income only with the 130% FPL free-meal threshold and missed Pennsylvania's universal free breakfast. That program makes breakfast free for all three children regardless of income, so free meal support is positive."
+us,scenario_028,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It ruled out free meals on income alone. It never applied Pennsylvania's universal free school breakfast (Act 33 of 2023), which gives the children free breakfast regardless of income."
+us,scenario_028,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It applied only the 130% FPL free-meal limit to $61,277 of income and missed Pennsylvania's universal free breakfast. That program puts breakfast in the free tier for the three children regardless of income."
+us,scenario_028,free_school_meals_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,It treated the 130% FPL limit as the only route to free meals and never applied Pennsylvania's universal free school breakfast. That program gives each child a free breakfast regardless of income.
+us,scenario_028,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,It applied only the federal 130% FPL free-meal threshold and missed Pennsylvania's Act 33 universal free breakfast. That program makes breakfast free for all three children regardless of income.
+us,scenario_028,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"It applied only the 130% FPL free-meal limit and missed Pennsylvania's universal free school breakfast. That program gives the three children free breakfast regardless of income, so free meal support is positive."
+us,scenario_028,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"It stopped at the 130% FPL income threshold and never applied Pennsylvania's universal free breakfast. That program gives every child a free breakfast regardless of income and produces $1,112.40 of free meal support."
+us,scenario_028,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,It treated the 130% poverty threshold as the only route to free meals and missed Pennsylvania's Act 33 universal free school breakfast. That program makes breakfast free for the three children regardless of income.
+us,scenario_028,free_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,It applied only the federal 130% FPL test and never applied Pennsylvania's universal free school breakfast. That program gives all three children a free breakfast regardless of household income.
+us,scenario_028,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no usable value or explanation for free_school_meals_eligible, so the answer is missing. The correct result is 1, because Pennsylvania's universal free school breakfast gives the three children free breakfast regardless of income."
+us,scenario_028,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,It compared income only with 130% of the poverty line and missed Pennsylvania's universal free school breakfast (Act 33 of 2023). That program gives the children a free breakfast regardless of income.
+us,scenario_028,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,It ruled out free meals on the income threshold alone and never applied Pennsylvania's universal free breakfast. That program gives all three children free breakfast regardless of income.
+us,scenario_028,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"It looked only for SNAP/TANF receipt or very low income and missed Pennsylvania's universal free school breakfast. That program makes breakfast free for all students at participating schools, these three children included, regardless of income."
+us,scenario_028,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,It checked only the federal free-meal income limit and direct certification and missed Pennsylvania's Act 33 universal free breakfast. That program gives each child a free breakfast regardless of income and produces positive free meal support.
+us,scenario_028,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,It applied only the federal free-meal income threshold and never applied Pennsylvania's universal free school breakfast. That program gives the three children free breakfast regardless of income.
+us,scenario_028,free_school_meals_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,It checked the 130% FPL limit and direct certification but missed Pennsylvania's universal free breakfast. That program puts breakfast in the free tier for all three children regardless of income.
+us,scenario_028,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,It ruled out free meals on the income threshold alone and missed Pennsylvania's Act 33 universal free school breakfast. That program gives each child a free breakfast regardless of income.
+us,scenario_028,free_school_meals_eligible,gpt-6-astra,llm_error,state_local_rule,False,It checked only the 130% FPL test and categorical eligibility and never applied Pennsylvania's universal free school breakfast. That program gives all three children free breakfast regardless of income.
+us,scenario_028,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,It applied the federal free-meal income limit and categorical eligibility but missed Pennsylvania's universal free breakfast. That program makes breakfast free for the three children regardless of household income.
+us,scenario_028,free_school_meals_eligible,gpt-6-sol,llm_error,state_local_rule,False,"It treated the federal income threshold and categorical eligibility as the only routes to free meals. It missed Pennsylvania's Act 33 universal free school breakfast, which gives each child a free breakfast regardless of income."
+us,scenario_028,free_school_meals_eligible,gpt-6.1-sol,llm_error,state_local_rule,False,"It compared $61,277 only with the four-person free-meal income limit and categorical eligibility. It missed Pennsylvania's universal free school breakfast, which gives the three children free breakfast worth $1,112.40 a year."
+us,scenario_028,free_school_meals_eligible,gpt-6.1-terra,llm_error,state_local_rule,False,It ruled out free meals on income alone and never applied Pennsylvania's universal free breakfast. That program gives all three children a free breakfast regardless of income.
+us,scenario_028,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,"It applied only the federal income threshold and missed Pennsylvania's Act 33 universal free school breakfast. That program makes breakfast free for each child regardless of income, so free meal support is positive."
+us,scenario_028,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"It counted income correctly at $61,277 but applied only the 130% FPL free-meal test. It never applied Pennsylvania's universal free breakfast, which gives the three children free breakfast regardless of income."
+us,scenario_028,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"It checked only the 130% FPL limit and categorical eligibility and missed Pennsylvania's universal free school breakfast. That program gives every child a free breakfast regardless of income, so PolicyEngine returns positive free meal support."
+us,scenario_028,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,It applied only the 130% FPL free-meal test and never applied Pennsylvania's Act 33 universal free breakfast. That program puts breakfast in the free tier for all three children regardless of income.
+us,scenario_028,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,It treated the federal 130% FPL threshold as the only route to free meals and missed Pennsylvania's universal free school breakfast. That program gives the three children free breakfast regardless of income.
+us,scenario_028,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,"It compared income only with 130% of FPL and missed Pennsylvania's universal free school breakfast. That program gives each child a free breakfast regardless of income and produces $1,112.40 of free meal support."
+us,scenario_028,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"It recognized only two routes to free meals, the 130% FPL limit and categorical eligibility, and missed Pennsylvania's universal free school breakfast. That program makes breakfast free for all three children regardless of income."
+us,scenario_028,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"It applied the 130% FPL cutoff and SNAP/TANF categorical eligibility but never applied Pennsylvania's Act 33 universal free breakfast. That program gives the children free breakfast regardless of income, so free meal support is positive."
+us,scenario_028,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,It applied only the federal 130% FPL free-meal threshold and missed Pennsylvania's universal free school breakfast. That program gives all three children a free breakfast regardless of income.
+us,scenario_028,free_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,It checked only the 130% FPL limit and categorical eligibility and never applied Pennsylvania's universal free breakfast. That program makes breakfast free for each child regardless of household income.
+us,scenario_028,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"It compared $61,277 only with 130% of poverty and missed Pennsylvania's Act 33 universal free school breakfast. That program gives the three children free breakfast regardless of income."
+us,scenario_028,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"It left the $1,267 of child support out of income and applied only the 130% FPL free-meal threshold. It missed Pennsylvania's universal free school breakfast, which gives all three children free breakfast regardless of income."
+us,scenario_028,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"It computed income correctly but applied only the federal 130% FPL free-meal threshold. It never applied Pennsylvania's universal free school breakfast, which gives each child a free breakfast regardless of income."
+us,scenario_028,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,It ruled out free meals on the federal income limit alone and missed Pennsylvania's Act 33 universal free breakfast. That program puts breakfast in the free tier for all three children regardless of income.
us,scenario_028,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_028,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_028,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning correctly summed $3,720 Social Security, $870 Medicare and $42 Pennsylvania employee UC (0.07% × $60,000) to $4,632, but it submitted 5,730. That number does not follow from any step it showed, so the submitted value contradicts its own correct derivation."
-us,scenario_028,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It counted only OASDI and Medicare ($4,590) and left out Pennsylvania's mandatory employee UC withholding of 0.07% on all wages, which adds $42 and brings the total to $4,632."
-us,scenario_028,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no employee-side payroll tax and mixed up corporate taxes with employee withholding. Pennsylvania withholds unemployment compensation tax from employees at 0.07% of all wages ($42 here), so the total is $4,632, not $4,590."
-us,scenario_028,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"It counted only employee FICA ($4,590) and never added Pennsylvania's mandatory employee UC contribution of 0.07% × $60,000 = $42."
-us,scenario_028,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It said Pennsylvania has no mandatory employee payroll tax. Pennsylvania withholds employee unemployment compensation tax at 0.07% of uncapped wages, which adds $42 for a total of $4,632."
-us,scenario_028,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"It stopped at Social Security plus Medicare ($4,590) and left out Pennsylvania's 0.07% employee UC withholding on the $60,000 of wages ($42)."
-us,scenario_028,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It counted only the 7.65% FICA ($4,590) and missed Pennsylvania's mandatory employee unemployment compensation withholding of 0.07% on all wages, which adds $42."
-us,scenario_028,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no separate employee payroll tax. Pennsylvania withholds UC tax from employees at 0.07% of gross wages with no wage cap, which adds $42 for a total of $4,632."
-us,scenario_028,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no employee UI contribution. Pennsylvania withholds unemployment compensation tax at 0.07% of all employee wages ($42 on $60,000), so the total is $4,632."
-us,scenario_028,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It said no Pennsylvania employee payroll tax is modeled, but the employee UC contribution of 0.07% × $60,000 = $42 is counted. The correct total is $4,632, not $4,590."
-us,scenario_028,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It said there is no Pennsylvania state employee payroll tax and so left out the mandatory 0.07% employee UC withholding on all $60,000 of wages ($42)."
-us,scenario_028,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It counted only Social Security and Medicare ($4,590) and never applied Pennsylvania's 0.07% employee unemployment compensation tax, which adds $42."
-us,scenario_028,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It said there is no state payroll tax and so dropped Pennsylvania's mandatory employee UC withholding of 0.07% × $60,000 = $42."
-us,scenario_028,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It counted only federal FICA ($4,590) and missed Pennsylvania's 0.07% employee unemployment compensation withholding on uncapped wages ($42)."
-us,scenario_028,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out Pennsylvania's mandatory 0.07% employee UC contribution ($42), so it reported $4,590 instead of $4,632."
-us,scenario_028,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It applied only the 7.65% federal FICA rate and missed Pennsylvania's employee UC withholding of 0.07% × $60,000 = $42."
-us,scenario_028,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"It counted only federal FICA ($4,590) and never added Pennsylvania's mandatory employee unemployment compensation tax of 0.07% on all wages ($42)."
-us,scenario_028,payroll_tax,gemini-4-argon,llm_error,state_local_rule,False,"It counted only Social Security and Medicare and left out Pennsylvania's 0.07% employee UC withholding ($42 on $60,000)."
-us,scenario_028,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It gave no usable payroll_tax value. The correct derivation is $3,720 Social Security + $870 Medicare + $42 Pennsylvania employee UC (0.07% of wages) = $4,632."
-us,scenario_028,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no mandatory employee state payroll tax. Pennsylvania withholds UC tax from employees at 0.07% of all wages, which adds $42 for a total of $4,632."
-us,scenario_028,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It said no mandatory state payroll tax applies in Pennsylvania. That leaves out the 0.07% employee unemployment compensation withholding on all $60,000 of wages ($42), so the total is $4,632."
-us,scenario_028,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"It added $3,720 and $870 wrongly, getting $4,600 instead of $4,590. It also left out Pennsylvania's 0.07% employee UC withholding ($42), which brings the correct total to $4,632."
-us,scenario_028,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"It explicitly excluded any Pennsylvania employee payroll tax. Pennsylvania's mandatory employee UC contribution of 0.07% × $60,000 = $42 belongs in the total, giving $4,632."
-us,scenario_028,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"It correctly included Pennsylvania's employee unemployment tax but computed only $7. That is 0.07% of the $10,000 employer UC taxable wage base. The employee withholding applies to all wages, so it is 0.07% × $60,000 = $42, for a total of $4,632."
-us,scenario_028,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,It applied only the 7.65% FICA rate and missed Pennsylvania's employee unemployment compensation withholding of 0.07% on uncapped wages ($42).
-us,scenario_028,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"It reasoned that no state payroll tax was specified, but Pennsylvania law requires employee UC withholding of 0.07% of wages ($42 here), so the total is $4,632."
-us,scenario_028,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only 7.65% FICA to $60,000 and left out Pennsylvania's mandatory employee UC contribution of 0.07% ($42)."
-us,scenario_028,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It wrongly said there is no Pennsylvania employee payroll tax. The 0.07% employee unemployment compensation withholding on all wages adds $42 to the $4,590 FICA, for $4,632."
-us,scenario_028,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It said Pennsylvania has no mandatory employee state payroll tax and so left out the 0.07% employee UC withholding on $60,000 ($42)."
-us,scenario_028,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no mandatory employee payroll tax. Pennsylvania withholds unemployment compensation tax from employees at 0.07% of gross wages, which adds $42."
-us,scenario_028,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It counted only Social Security and Medicare ($4,590) and never applied Pennsylvania's 0.07% employee UC tax on the $60,000 of wages ($42)."
-us,scenario_028,payroll_tax,inkling,llm_error,state_local_rule,False,"It counted only federal FICA and missed Pennsylvania's mandatory employee unemployment compensation withholding of 0.07% × $60,000 = $42."
-us,scenario_028,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no usable payroll_tax value. The correct total is $3,720 Social Security + $870 Medicare + $42 Pennsylvania employee UC (0.07% of $60,000) = $4,632."
-us,scenario_028,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"It said no PolicyEngine-modeled employee state payroll tax applies in Pennsylvania, but the PA employee UC contribution of 0.07% of wages ($42) is modeled and counted."
-us,scenario_028,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no mandatory employee state payroll tax and so left out the 0.07% employee UC withholding ($42) on the $60,000 of wages."
-us,scenario_028,payroll_tax,muse-spark-1.3,llm_error,state_local_rule,False,"It counted only Social Security and Medicare ($4,590) and never added Pennsylvania's 0.07% employee unemployment compensation tax, which adds $42."
-us,scenario_028,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It stopped at federal FICA and missed Pennsylvania's mandatory employee UC withholding of 0.07% on uncapped wages ($42), so it reported $4,590 instead of $4,632."
-us,scenario_028,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It wrongly said Pennsylvania deducts no unemployment tax from employees. Pennsylvania withholds UC tax from employee wages at 0.07% with no cap, which adds $42 for $4,632."
-us,scenario_028,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"It applied only the 7.65% FICA rate and left out Pennsylvania's 0.07% employee unemployment compensation contribution on $60,000 ($42)."
+us,scenario_028,payroll_tax,claude-fable-5,llm_error,other,False,"Its working got the right total: $3,720 Social Security + $870 Medicare + $42 Pennsylvania employee UC (0.07% × $60,000) = $4,632. It then submitted 5,730, a number its own working never produced, so the final value contradicts the derivation."
+us,scenario_028,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only employee OASDI and Medicare ($4,590). It left out Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% on all wages, which adds $42 and gives $4,632."
+us,scenario_028,payroll_tax,claude-fable-5.2,llm_error,state_local_rule,False,"Added only Social Security ($3,720) and Medicare ($870). It never included Pennsylvania's employee UC contribution of 0.07% × $60,000 = $42, which brings the total to $4,632."
+us,scenario_028,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Said Pennsylvania has no employee-side payroll tax because it confused it with the state's corporate taxes. Pennsylvania withholds a mandatory employee UC contribution of 0.07% of gross wages, which is $42 here. The total should be $4,632, not $4,590."
+us,scenario_028,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"Stopped at federal FICA ($4,590). It omitted Pennsylvania's mandatory 0.07% employee unemployment compensation withholding on all wages ($42), which belongs in the employee payroll tax total of $4,632."
+us,scenario_028,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Said Pennsylvania has no mandatory employee payroll tax. Pennsylvania's employee UC contribution of 0.07% of uncapped wages adds $42 to the $4,590 of FICA, for $4,632."
+us,scenario_028,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Computed only Social Security and Medicare ($4,590). It missed the Pennsylvania employee unemployment compensation contribution (0.07% × $60,000 = $42), which makes the total $4,632."
+us,scenario_028,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"Applied only the 7.65% FICA rate. It left out Pennsylvania's mandatory employee UC withholding of 0.07% on all wages ($42), so it reported $4,590 instead of $4,632."
+us,scenario_028,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"Said Pennsylvania has no separate employee payroll tax beyond its flat income tax. Pennsylvania requires an employee unemployment compensation contribution of 0.07% of gross wages ($42 here), which brings the total to $4,632."
+us,scenario_028,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Said Pennsylvania has no employee UI contribution. Pennsylvania is one of the few states that withholds employee UC, at 0.07% of all wages ($42 on $60,000), so the total is $4,632, not $4,590."
+us,scenario_028,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"Concluded that no Pennsylvania employee payroll tax is modeled. Pennsylvania's 0.07% employee UC contribution on $60,000 adds $42 to the $4,590 of FICA, for $4,632."
+us,scenario_028,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Said Pennsylvania has no state employee payroll tax. Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% of uncapped wages ($42) belongs in the total, which makes it $4,632."
+us,scenario_028,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"Included only Social Security and Medicare ($4,590). It omitted Pennsylvania's 0.07% employee UC withholding on the $60,000 of wages ($42), so the total is $4,632."
+us,scenario_028,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Stated that no state payroll tax applies. Pennsylvania requires an employee unemployment compensation contribution of 0.07% of all wages ($42), which raises the total from $4,590 to $4,632."
+us,scenario_028,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Summed only federal Social Security and Medicare ($4,590). It missed Pennsylvania's mandatory employee UC contribution of 0.07% × $60,000 = $42, which makes the total $4,632."
+us,scenario_028,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Computed only federal FICA ($4,590). It never added Pennsylvania's employee unemployment compensation withholding of 0.07% on all wages ($42), so the total is $4,632."
+us,scenario_028,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Applied only the 7.65% FICA rate. It omitted Pennsylvania's mandatory 0.07% employee UC contribution ($42 on $60,000) that the question's definition of mandatory employee state payroll taxes includes."
+us,scenario_028,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,"Treated employee payroll tax as FICA alone ($4,590). It left out Pennsylvania's employee UC withholding of 0.07% of gross wages ($42), which makes the total $4,632."
+us,scenario_028,payroll_tax,gemini-4-argon,llm_error,state_local_rule,False,"Computed only Social Security and Medicare. It missed Pennsylvania's mandatory employee unemployment compensation contribution of 0.07% × $60,000 = $42, which brings the total to $4,632."
+us,scenario_028,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"Returned no usable payroll_tax value or explanation. The correct derivation is $3,720 Social Security + $870 Medicare + $42 Pennsylvania employee UC = $4,632."
+us,scenario_028,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"Said Pennsylvania has no mandatory employee state payroll tax. Pennsylvania withholds employee UC at 0.07% of all wages ($42), so the total is $4,632, not $4,590."
+us,scenario_028,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Said no mandatory state payroll tax applies in Pennsylvania. Pennsylvania's 0.07% employee unemployment compensation contribution on uncapped wages adds $42 to the $4,590 of FICA."
+us,scenario_028,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Its own formula gives $3,720 + $870 = $4,590, but it misadded the sum to $4,600. It also omitted Pennsylvania's 0.07% employee UC contribution ($42), so the correct total is $4,632."
+us,scenario_028,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"Deliberately left out any Pennsylvania employee payroll tax. Pennsylvania's mandatory employee UC contribution of 0.07% on all wages ($42) is a mandatory employee state payroll tax, so the total is $4,632."
+us,scenario_028,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"Correctly included Pennsylvania's employee UC contribution but computed it as $7, which is 0.07% of only the $10,000 employer taxable wage base. The employee contribution applies to total wages with no cap: 0.07% × $60,000 = $42, for a total of $4,632."
+us,scenario_028,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"Applied only 7.65% FICA to $60,000. It omitted Pennsylvania's mandatory 0.07% employee UC withholding ($42), so the total is $4,632."
+us,scenario_028,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,"Treated the lack of a stated state payroll tax as zero. Pennsylvania's statutory employee UC contribution of 0.07% of all wages applies automatically ($42), so the total is $4,632."
+us,scenario_028,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Applied only 7.65% FICA. It never added Pennsylvania's employee unemployment compensation contribution of 0.07% × $60,000 = $42, so the total is $4,632."
+us,scenario_028,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"Said Pennsylvania has no employee payroll tax. Pennsylvania withholds 0.07% of gross wages for employee UC ($42), which brings the total from $4,590 to $4,632."
+us,scenario_028,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"Said Pennsylvania has no mandatory employee state payroll tax. The 0.07% Pennsylvania employee UC contribution on uncapped wages ($42) belongs in the total of $4,632."
+us,scenario_028,payroll_tax,grok-4.7,llm_error,state_local_rule,False,"Concluded that Pennsylvania has no mandatory employee state payroll tax. Pennsylvania's employee unemployment compensation contribution of 0.07% of all wages adds $42, for $4,632."
+us,scenario_028,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"Computed only Social Security and Medicare ($4,590). It missed Pennsylvania's mandatory employee UC withholding of 0.07% × $60,000 = $42, so the total is $4,632."
+us,scenario_028,payroll_tax,inkling,llm_error,state_local_rule,False,"Summed only federal FICA. It omitted Pennsylvania's 0.07% employee unemployment compensation contribution on the head's $60,000 of wages ($42), so the total is $4,632."
+us,scenario_028,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no usable payroll_tax value or explanation. The correct derivation is $3,720 Social Security + $870 Medicare + $42 Pennsylvania employee UC = $4,632."
+us,scenario_028,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"Said PolicyEngine models no employee state payroll tax for Pennsylvania. PolicyEngine includes Pennsylvania's employee UC contribution of 0.07% of wages ($42), which makes the total $4,632."
+us,scenario_028,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"Said Pennsylvania has no mandatory employee state payroll tax. Pennsylvania's 0.07% employee unemployment compensation withholding on all wages ($42) brings the total from $4,590 to $4,632."
+us,scenario_028,payroll_tax,muse-spark-1.3,llm_error,state_local_rule,False,"Included only Social Security and Medicare. It left out Pennsylvania's mandatory employee UC contribution of 0.07% × $60,000 = $42, so the total is $4,632."
+us,scenario_028,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"Computed FICA alone ($4,590). It never applied Pennsylvania's employee unemployment compensation contribution of 0.07% on uncapped wages ($42), so the total is $4,632."
+us,scenario_028,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Explicitly said Pennsylvania has no unemployment deduction from employees. Pennsylvania withholds an employee UC contribution of 0.07% of all gross wages ($42 here), so the total is $4,632."
+us,scenario_028,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Applied only the 7.65% FICA rate. It omitted Pennsylvania's mandatory 0.07% employee UC contribution ($42), which brings the total to $4,632."
us,scenario_028,reduced_price_school_meals_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model counted the $61,277 of income correctly and first found it above the 185% limit using the 2025 guideline. It then switched to a made-up uprated 2026 guideline that put the limit at about $61,300. With the actual 2026 guideline of $33,000 for four, the limit is $61,050, so $61,277 is over it and the model's invented uprating flipped the answer to Yes."
us,scenario_028,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model tested only wages plus interest ($60,010) and left out the $1,267 of child support, which counts as income for school meals. With child support included, income is $61,277, which is above the model's own 185% limit of $61,050."
us,scenario_028,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model counted $61,277 of income correctly but put the 185% limit at about $61,975, which implies a guideline of roughly $33,500. The 2026 guideline for four is $33,000, so the reduced-price limit is $61,050 and the household is over it."
@@ -3001,57 +3042,58 @@ us,scenario_030,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_outpu
us,scenario_030,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_030,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model miscalculated Medicare tax: 1.45% of $13,000 is $188.50, not $189 or $189.90. It then compounded the arithmetic error by stating that $806 plus $189 equals $995.90; the exact component sum is $994.50."
us,scenario_030,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Applied only the 130% FPL gross income test and denied eligibility. Texas BBCE (TANF non-cash) sets the gross limit at 165% FPL, and this household's 1.60× ratio passes it, which makes the household categorically eligible. As an eligible one-person household it gets the $24/month minimum allotment, or $288/year."
-us,scenario_030,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Correctly computed net income of about $1,658 and that 30% of it exceeds $298, but then denied benefits on the net income test. Texas BBCE categorical eligibility waives the net income test, and eligible 1–2 person households get the minimum allotment ($24/month = $288/year) even when the contribution exceeds the maximum."
-us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Counted the excluded $5,000 educational assistance in a $30,000 income total and applied a 130% FPL test with a wrong $1,868 threshold. It missed that Texas BBCE (165% FPL) makes the household categorically eligible at 1.60× FPL, which entitles it to the $24/month minimum allotment."
-us,scenario_030,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Applied gross and net income tests and concluded there is no benefit. Wages plus financial assistance equal 1.60× FPL, which is under Texas's 165% BBCE limit, so the household is categorically eligible and gets the $24/month minimum allotment ($288/year)."
-us,scenario_030,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"Recognized Texas BBCE at 165% FPL but miscomputed the limit as about $2,072. The correct figure is 165% × $1,304.17 = $2,152, so the $2,083 gross income passes. The household is categorically eligible, and the net-income result yields the $24/month minimum allotment rather than $0."
-us,scenario_030,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Excluded the $12,000 of financial assistance from SNAP income, but SNAP counts it as unearned income ($1,000/month). With it included, net income is $1,658 and the $498 contribution exceeds $298, so only the $24 minimum is paid. The model's $2,298 also contradicts its own ~$99/month figure."
-us,scenario_030,snap,claude-opus-5,llm_error,categorical_eligibility,False,"Counted only the $13,000 of wages and left out the $12,000 of financial assistance, which SNAP counts as unearned income. Including it raises net income to $1,658/month, and the 30% contribution ($498) exceeds the $298 maximum, which leaves the $24 minimum allotment."
-us,scenario_030,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"Explicitly assumed the financial assistance is not SNAP income. SNAP counts the $12,000 as unearned income, which raises net income from $658 to $1,658/month. The $298 − $498 result is negative, so the benefit is the $24 minimum allotment, not about $101."
-us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Wrongly stated that Texas has no broad-based categorical eligibility. Texas confers TANF non-cash categorical eligibility up to 165% FPL, and this household's 1.60× FPL gross income passes it, so it is eligible for the $24/month minimum allotment."
-us,scenario_030,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"Excluded the $12,000 of financial assistance from countable income, and its own formula ($298 − ~$200 ≈ $98) contradicts the $350/month it reported. Counting the financial assistance gives net income of $1,658 and a contribution of $498 above the maximum, so the benefit is the $24 minimum."
-us,scenario_030,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Treated the financial assistance as excluded, but SNAP counts the $12,000 as unearned income. Net income is then $1,658/month, the contribution ($498) exceeds the $298 maximum, and the household gets only the $24 minimum allotment."
-us,scenario_030,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Counted the $5,000 educational assistance, which SNAP excludes, and applied the 130% gross test. Correct gross income is $2,083/month (1.60× FPL), which is under Texas's 165% BBCE limit, so the household is categorically eligible and gets the $24/month minimum."
-us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Included the excluded educational assistance to reach $30,000 and applied the federal 130% gross limit. It missed Texas BBCE at 165% FPL, which the $25,000 countable income passes, so the household is eligible for the $24/month minimum allotment."
-us,scenario_030,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Counted the excluded educational assistance, then stopped once the contribution exceeded the one-person maximum allotment. Eligible one-person households get the minimum allotment (8% of $298 ≈ $24/month) rather than $0."
-us,scenario_030,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Counted the pregnant head as a two-person SNAP household, but SNAP does not count an unborn child, so the unit size is 1 with a $298 maximum allotment. With the correct $1,658 net income, the $498 contribution exceeds $298 and the benefit is the $24 minimum in place of the model's $49."
-us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"Correctly used household size 1 and $25,000 of countable income but applied the 130% gross limit. Texas BBCE raises that limit to 165% FPL, and 1.60× FPL passes, so the household is categorically eligible for the $24/month minimum allotment."
-us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Based the benefit on the $13,000 of earnings alone and left out the $12,000 of financial assistance, which SNAP counts as unearned income. Including it makes net income $1,658/month, which leaves only the $24 minimum allotment instead of about $217/month."
-us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross test and denied eligibility. Texas BBCE at 165% FPL covers this household's 1.60× FPL gross income, which makes it categorically eligible for the $24/month minimum allotment."
-us,scenario_030,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"Used Texas's 165% BBCE limit but put it at $24,849. The correct limit is 165% × the $15,650 guideline ≈ $25,822, so $25,000 of income passes. The household is categorically eligible and gets the $24/month minimum allotment."
-us,scenario_030,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Asserted $0 without reasoning, which treats the household as ineligible. Countable income of 1.60× FPL passes Texas's 165% BBCE limit, and the eligible one-person household gets the $24/month minimum allotment ($288/year)."
-us,scenario_030,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"Denied benefits on net income limits, but Texas BBCE categorical eligibility waives the net income test. The eligible one-person household receives the $24/month minimum allotment even though 30% of net income exceeds the maximum."
-us,scenario_030,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Treated net income above the threshold as disqualifying, but categorical eligibility through Texas BBCE (gross income 1.60× FPL < 165%) bypasses the net income test. That entitles the household to the $24/month minimum allotment."
-us,scenario_030,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,Correctly counted the financial assistance but concluded that no positive allotment exists once the contribution exceeds the maximum. Categorically eligible 1–2 person households receive the minimum allotment of about $24/month ($288/year).
-us,scenario_030,snap,gemini-4-argon,llm_error,categorical_eligibility,False,"Counted only wages, leaving out the $12,000 of financial assistance that SNAP counts as unearned income, and used an outdated $292 maximum instead of $298. The correct net income of $1,658 gives a $498 contribution above the maximum, so the benefit is the $24 minimum."
-us,scenario_030,snap,glm-5.2,llm_error,categorical_eligibility,False,"Counted the unborn child to make a two-person household and took a medical deduction for a non-elderly, non-disabled head; pregnancy does not qualify for the medical deduction. With size 1 and no medical deduction, net income is $1,658, which leaves only the $24/month minimum allotment."
-us,scenario_030,snap,glm-5.3,llm_error,categorical_eligibility,False,"Found the household over the gross and net income limits but missed Texas BBCE. At 1.60× FPL, gross income is under the 165% limit, so both standard income tests are waived and the household receives the $24/month minimum allotment."
-us,scenario_030,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Treated countable cash income as very low, which leaves out the $12,000 of financial assistance that SNAP counts as unearned income. With it included, net income is $1,658/month and the benefit falls to the $24 minimum, not about $176/month."
-us,scenario_030,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Gave $0 with no income test, which treats the household as ineligible. Countable gross income of 1.60× FPL passes Texas's 165% BBCE limit, and the eligible one-person household gets the $24/month minimum allotment."
-us,scenario_030,snap,gpt-5.5,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income ($498) exceeds the $298 maximum but then paid nothing. It skipped the minimum allotment rule, under which eligible one- and two-person households receive 8% of the one-person maximum, about $24/month."
-us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"Applied a one-person gross income limit without Texas BBCE. Under the 165% FPL BBCE limit, gross income of 1.60× FPL passes, and the household gets the $24/month minimum allotment."
-us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"Counted income correctly (wages plus financial assistance, excluding educational assistance) but wrongly found it above Texas's gross limit. Texas BBCE allows up to 165% FPL, the household is at 1.60×, and it is therefore eligible for the $24/month minimum."
-us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Compared $25,000 against the standard 130% gross limit. Texas BBCE raises the limit to 165% FPL (about $25,822 annualized), which makes the household categorically eligible for the $24/month minimum allotment."
-us,scenario_030,snap,gpt-6-astra,llm_error,categorical_eligibility,False,"Applied the standard income limits, but Texas BBCE categorical eligibility (gross income 1.60× FPL < 165%) waives them. The eligible one-person household receives the $24/month minimum allotment."
-us,scenario_030,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"Correctly found the 30% contribution above the one-person maximum but returned $0. It ignored the minimum allotment for eligible one- and two-person households, which is about $24/month ($288/year)."
-us,scenario_030,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"Based net income on wages alone and left out the $12,000 of financial assistance, which SNAP counts as unearned income. Including it puts net income at $1,658, the contribution exceeds $298, and the benefit is the $24 minimum allotment."
-us,scenario_030,snap,gpt-6.1-sol,llm_error,categorical_eligibility,False,"Concluded there is no positive allotment because countable income is too high. Eligible one-person households have a guaranteed minimum allotment of about $24/month, so the annual amount is $288, not $0."
-us,scenario_030,snap,gpt-6.1-terra,llm_error,categorical_eligibility,False,"Denied eligibility on standard income limits. Texas BBCE confers categorical eligibility at 1.60× FPL gross income, which waives those limits and yields the $24/month minimum allotment."
-us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"Gave $0 with no stated test, which treats the household as ineligible. The $190 in assets passes and gross income passes Texas's 165% BBCE limit, so the household is eligible for the $24/month minimum allotment."
-us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross income test to $25,000 of countable income. Texas BBCE uses a 165% FPL limit that this income passes, which makes the household categorically eligible for the $24/month minimum."
-us,scenario_030,snap,grok-4.6,llm_error,categorical_eligibility,False,"Wrongly asserted that income exceeds the Texas BBCE 165% FPL limit, and hedged toward counting educational assistance, which SNAP excludes. Countable income of $2,083/month is 1.60× the $1,304.17 guideline, so the household is categorically eligible for the $24/month minimum."
-us,scenario_030,snap,grok-4.7,llm_error,categorical_eligibility,False,"Computed net income of about $19,900/year correctly but denied benefits on the 100% FPL net income test. Texas BBCE categorical eligibility waives the net income test, and the household receives the $24/month minimum allotment."
-us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"Applied the federal 130% gross limit, overstated at $20,800. It missed Texas BBCE at 165% FPL, which $25,000 of countable income passes, so the household is eligible for the $24/month minimum allotment."
-us,scenario_030,snap,inkling,llm_error,categorical_eligibility,False,"Counted the $5,000 educational assistance, which SNAP excludes, and applied the 130% gross test. Correct gross income is 1.60× FPL, which is under Texas's 165% BBCE limit, so the household gets the $24/month minimum."
-us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there was no answer to score. The correct derivation gives a $24/month minimum allotment, or $288/year."
-us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,"Correctly computed $2,083.33/month of countable gross income but compared it to the standard 130% limit. Texas BBCE uses 165% FPL ($2,152), so the household is categorically eligible for the $24/month minimum allotment."
-us,scenario_030,snap,minimax-m3,llm_error,categorical_eligibility,False,"Compared an annual figure ($10,400) with a monthly net income limit and left out the financial assistance. It also missed Texas BBCE, which waives the net income test, and the $24/month minimum allotment for eligible one-person households."
-us,scenario_030,snap,muse-spark-1.3,llm_error,categorical_eligibility,False,"Applied the 130% FPL gross test to $25,000 of countable income. Texas BBCE raises the gross limit to 165% FPL, which this income passes, so the household gets the $24/month minimum allotment."
-us,scenario_030,snap,muse-wonder,llm_error,categorical_eligibility,False,"Denied benefits because net income is above the limit, but Texas BBCE categorical eligibility waives the net income test. The eligible one-person household receives the $24/month minimum allotment."
-us,scenario_030,snap,ox-alpha,llm_error,categorical_eligibility,False,"Correctly counted $25,000 of income but applied the federal 130% gross limit. Texas BBCE at 165% FPL (about $25,822) makes the household categorically eligible, which yields the $24/month minimum allotment."
-us,scenario_030,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Left out the $12,000 of financial assistance (countable unearned income) and skipped the standard deduction. Its reported $2,576 also contradicts its own $31–32/month arithmetic. The correct $1,658 net income gives a contribution above the maximum and a $24/month minimum."
-us,scenario_030,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"Counted the unborn child to make a two-person household and claimed $0 net income, ignoring $2,083/month of countable income. Its reported $3,576 also conflicts with its own $4,272. The correct one-person unit with $1,658 net income receives only the $24/month minimum allotment."
+us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model correctly found $2,083/month in countable income, then rejected the household under the 130% FPL gross test. It never applied Texas BBCE (TANF non-cash), which allows gross income up to 165% FPL; this household is at 1.6×. As an eligible one-person household, it gets the $24/month minimum allotment, or $288 a year."
+us,scenario_030,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"The model correctly computed $1,658 in net income and found that 30% of it is more than the $298 maximum. It wrongly applied the 100% FPL net income test, which BBCE waives in Texas, and it skipped the $24 minimum allotment for an eligible one-person household. The correct annual result is $288."
+us,scenario_030,snap,claude-fable-5.2,llm_error,categorical_eligibility,False,"The model left out the $12,000 in financial assistance, which SNAP counts as unearned income ($1,000/month). That understated net income at $657.67 instead of $1,658. With the correct figure, the $498 contribution is more than the $298 maximum, and only the $24/month minimum allotment remains."
+us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model counted the excluded $5,000 in educational assistance and tested income only against the 130% FPL gross limit. Texas BBCE raises that limit to 165% FPL, and the household's actual countable income ($2,083/month, 1.6× FPL) passes it. The model also missed the $24/month minimum allotment that an eligible one-person household receives."
+us,scenario_030,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model asserted $0 without applying any rules. The household is categorically eligible through Texas BBCE (gross income 1.6× FPL, under the 165% limit). Its $498 contribution exceeds the $298 maximum, so it gets the $24/month minimum allotment, or $288 a year."
+us,scenario_030,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly invoked Texas BBCE at 165% FPL but used the outdated $15,060 guideline, which gave a limit of about $2,072. Under the $15,650 guideline in effect for 2026, 165% is $2,152/month, so $2,083 passes. The model also missed the resulting $24/month minimum allotment."
+us,scenario_030,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model excluded the $12,000 in financial assistance from countable income, even though SNAP counts it as unearned income. The model also contradicted its own calculation of about $99/month by reporting nearly the full maximum allotment. With the assistance counted, net income is $1,658 and only the $24/month minimum allotment applies."
+us,scenario_030,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model counted only the $13,000 in wages and dropped the $12,000 in financial assistance, which is countable unearned income. That produced a benefit of about $134/month. The correct net income of $1,658 gives a $498 contribution, which exceeds the maximum, so the $24/month minimum allotment applies."
+us,scenario_030,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model explicitly assumed the financial assistance was not countable. SNAP counts the $12,000 as unearned income, which raises net income from $657.67 to $1,658. That makes the contribution $498 and leaves only the $24/month minimum allotment."
+us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model wrongly stated that Texas has no broad-based categorical eligibility and applied the 130% FPL gross test. Texas confers categorical eligibility through TANF non-cash benefits with a 165% FPL gross limit, and this household passes at 1.6×. The household is therefore eligible for the $24/month minimum allotment."
+us,scenario_030,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model left the $12,000 in financial assistance out of countable income. It then made an arithmetic error: its own figures ($298 minus about $200) give about $98, but it reported $350/month. With the assistance counted, the contribution is $498 and only the $24/month minimum applies."
+us,scenario_030,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"The model treated the $12,000 in financial assistance as excluded, but SNAP counts it as unearned income. Counting it raises net income to $1,658 and the contribution to $498, so the household gets only the $24/month minimum allotment."
+us,scenario_030,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"The model counted the excluded $5,000 in educational assistance and applied the 130% FPL gross test. Countable income is $2,083/month, which is 1.6× FPL and under Texas's 165% BBCE limit. The household is therefore eligible for the $24/month minimum allotment."
+us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model included the excluded educational assistance, giving $30,000, and tested it against the 130% FPL limit. Texas BBCE uses a 165% limit, and the correct countable income of $25,000 (1.6× FPL) passes it. The eligible one-person household receives the $24/month minimum allotment."
+us,scenario_030,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income is more than the one-person maximum allotment, but then set the benefit to $0. It skipped the $24/month minimum allotment for eligible one- and two-person households. It also wrongly counted the excluded educational assistance."
+us,scenario_030,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model counted the pregnant head as a two-person SNAP household and used the $546 maximum allotment. An unborn child does not add a member, so the household is size 1 with a $298 maximum. With the model's own $497 contribution, the benefit falls to the $24/month minimum allotment."
+us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model correctly used household size 1 and $25,000 in countable income, then tested it against the 130% FPL gross limit. Texas BBCE uses a 165% limit, about $25,823 a year, and the household passes it. The eligible household receives the $24/month minimum allotment."
+us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model based the benefit only on the $13,000 in earnings and left out the $12,000 in financial assistance, which is countable unearned income. Including it makes the contribution $498, which is more than the $298 maximum, so the benefit is the $24/month minimum rather than $217/month."
+us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model applied the 130% FPL gross income test. Texas BBCE replaces it with a 165% limit, and this household's gross income is 1.6× FPL. The household is eligible and receives the $24/month minimum allotment."
+us,scenario_030,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model correctly applied Texas's 165% BBCE limit but computed it from the outdated $15,060 guideline, getting $24,849. Under the $15,650 guideline, the limit is $25,823, so $25,000 passes. The household gets the $24/month minimum allotment."
+us,scenario_030,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model gave a bare $0 without applying any rules. The household passes Texas BBCE at 1.6× FPL, and its $498 contribution exceeds the $298 maximum. The $24/month minimum allotment therefore applies, for $288 a year."
+us,scenario_030,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model concluded that net income leaves no positive allotment. Texas BBCE waives the net income test, and an eligible one-person household whose contribution exceeds the maximum still receives the $24/month minimum allotment."
+us,scenario_030,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model treated net income above the threshold as disqualifying. BBCE waives the net income test, and the one-person household receives the $24/month minimum allotment even though its $498 contribution is more than the $298 maximum."
+us,scenario_030,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"The model correctly counted the financial assistance and found that the contribution exceeds the maximum allotment, but it stopped at $0. It missed the $24/month minimum allotment for eligible one-person households, which gives $288 a year."
+us,scenario_030,snap,gemini-4-argon,llm_error,categorical_eligibility,False,"The model counted only earnings and left out the $12,000 in financial assistance, which is countable unearned income. It also used an outdated $292 maximum allotment instead of $298. With the assistance counted, net income is $1,658 and only the $24/month minimum applies."
+us,scenario_030,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model treated pregnancy as making a two-person SNAP household; the unborn child does not count, so the household is size 1 with a $298 maximum. It also took a medical expense deduction, which is available only to elderly or disabled members. Without that deduction, net income is $1,658 and the benefit is the $24/month minimum."
+us,scenario_030,snap,glm-5.3,llm_error,categorical_eligibility,False,"The model applied the standard gross and net limits. Texas BBCE waives the net test and sets the gross limit at 165% FPL, which this household passes at 1.6×. The eligible one-person household receives the $24/month minimum allotment."
+us,scenario_030,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated countable income as very low, ignoring the $12,000 in financial assistance, which SNAP counts as unearned income. With it counted, net income is $1,658 and the contribution is $498, so only the $24/month minimum allotment applies, not about $176/month."
+us,scenario_030,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model gave $0 without any income computation. The household is categorically eligible through Texas BBCE (1.6× FPL, under 165%). Its $498 contribution exceeds the maximum, so it gets the $24/month minimum allotment."
+us,scenario_030,snap,gpt-5.5,llm_error,categorical_eligibility,False,The model correctly found that 30% of net income exceeds the $298 one-person maximum. It then set the benefit to $0 instead of the $24/month minimum allotment guaranteed to eligible one- and two-person households.
+us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model applied the regular gross income limit. Texas BBCE through TANF non-cash eligibility sets the gross limit at 165% FPL, and this household passes at 1.6×. It therefore gets the $24/month minimum allotment."
+us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model correctly counted $25,000 and excluded educational assistance, but then said the household is over the Texas gross limit. Texas's BBCE limit is 165% FPL (about $25,823 under the $15,650 guideline), which the household passes. It receives the $24/month minimum allotment."
+us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model tested $25,000 against the standard one-person gross limit. Texas BBCE raises that limit to 165% FPL, which the household passes at 1.6×, so it receives the $24/month minimum allotment."
+us,scenario_030,snap,gpt-6-astra,llm_error,categorical_eligibility,False,"The model applied the standard SNAP income limits. Texas BBCE uses a 165% gross limit and waives the net test, so the household is eligible. With its contribution above the maximum, it receives the $24/month minimum allotment."
+us,scenario_030,snap,gpt-6-luna,llm_error,categorical_eligibility,False,The model correctly found that the 30% net income contribution exceeds the one-person maximum allotment. It then returned $0 instead of the $24/month minimum allotment that eligible one-person households receive.
+us,scenario_030,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"The model computed net income from wages alone and dropped the $12,000 in financial assistance, which is countable unearned income. Including it makes net income $1,658 and the contribution $498, which is above the maximum, so the benefit is the $24/month minimum."
+us,scenario_030,snap,gpt-6.1-sol,llm_error,categorical_eligibility,False,"The model correctly reasoned that countable income leaves no positive formula benefit. It missed the $24/month minimum allotment that applies to an eligible one-person household, which gives $288 a year."
+us,scenario_030,snap,gpt-6.1-terra,llm_error,categorical_eligibility,False,"The model applied the standard SNAP income limits. Texas BBCE waives the net test and allows gross income up to 165% FPL, which this household meets at 1.6×. The household gets the $24/month minimum allotment."
+us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model asserted $0 without any computation. The household passes Texas BBCE at 1.6× FPL and gets the $24/month minimum allotment, because its $498 contribution is more than the $298 maximum."
+us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model correctly counted $25,000 but tested it against the 130% FPL gross limit. Texas BBCE uses 165% FPL, which the household passes, so it receives the $24/month minimum allotment."
+us,scenario_030,snap,grok-4.6,llm_error,categorical_eligibility,False,"The model claimed income exceeds the 165% BBCE limit by counting educational assistance, which is excluded. Countable income of $25,000 is 1.6× the $15,650 guideline, under 165%, so the household is eligible for the $24/month minimum allotment."
+us,scenario_030,snap,grok-4.7,llm_error,categorical_eligibility,False,"The model correctly built net income but applied the 100% FPL net income test, which Texas BBCE waives for categorically eligible households. The eligible one-person household receives the $24/month minimum allotment."
+us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model used the 130% FPL gross limit of about $20,800. Texas BBCE uses 165% FPL, about $25,823, and $25,000 passes. The household receives the $24/month minimum allotment."
+us,scenario_030,snap,inkling,llm_error,categorical_eligibility,False,"The model counted the excluded $5,000 in educational assistance and applied the 130% FPL gross test. Texas BBCE uses 165% FPL, and countable income of $25,000 passes it, so the household gets the $24/month minimum allotment."
+us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value and no reasoning, so there was no answer to score against the $288 reference."
+us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model correctly computed $2,083.33/month but compared it to the standard 130% gross limit. Texas BBCE allows 165% FPL ($2,152/month), so the household is eligible and receives the $24/month minimum allotment."
+us,scenario_030,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model compared annual income after the earned income deduction to a monthly net limit, mixing time periods. It also applied a net income test that Texas BBCE waives. The categorically eligible one-person household receives the $24/month minimum allotment."
+us,scenario_030,snap,muse-spark-1.3,llm_error,categorical_eligibility,False,"The model tested $25,000 against the 130% FPL gross limit. Texas BBCE raises that to 165% FPL, which the household passes at 1.6×, so it gets the $24/month minimum allotment."
+us,scenario_030,snap,muse-wonder,llm_error,categorical_eligibility,False,"The model treated net income above the limit as disqualifying. Texas BBCE waives the net income test, and the eligible one-person household receives the $24/month minimum allotment despite its $498 contribution."
+us,scenario_030,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model correctly counted $25,000 but applied the 130% FPL limit of about $20,350. Texas BBCE uses 165% FPL, which the household passes, so it receives the $24/month minimum allotment."
+us,scenario_030,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model omitted the $12,000 in financial assistance and the $209 standard deduction. Its final $2,576 does not follow from its own monthly figures of $31 to $32. The correct net income of $1,658 gives a $498 contribution, so the benefit is the $24/month minimum."
+us,scenario_030,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model counted the unborn child to make a two-person household and claimed net income of $0. The SNAP household is size 1, and countable income of $2,083/month gives $1,658 in net income. The resulting contribution exceeds the $298 maximum, leaving only the $24/month minimum allotment."
us,scenario_030,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_030,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no TANF output, violating the required structured-output contract rather than making a substantive benefit calculation."
us,scenario_030,tanf,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly counted the unborn child as a current dependent child and constructed a two-person Texas TANF assistance unit. A pregnant adult with no born dependent child does not satisfy the family-with-dependent-children requirement, so the $271 monthly grant schedule does not apply."
@@ -3059,102 +3101,104 @@ us,scenario_031,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_er
us,scenario_031,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly denied the listed alimony deduction and used an estimated $10,207 post-TCJA-reversion standard deduction. PolicyEngine deducts the $1,165 alimony expense and applies the applicable 2026 single age-65-or-older standard deduction, leaving no taxable income."
us,scenario_031,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own calculation produced zero taxable income, zero tax, and zero nonrefundable credits, but it submitted $1,268.50. Its numeric output directly contradicts every computation step in its explanation."
us,scenario_031,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $5,357.50 of Social Security as taxable even though provisional income is below the $25,000 single-filer base, and it then failed to follow its own calculation that the elderly credit reduced tax to zero. The alimony adjustment yields $13,243 of AGI, the standard deduction eliminates taxable income, and the submitted $2,439 has no support in its stated arithmetic."
-us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,state_local_rule,False,"It subtracted the $1,165 alimony expense, which SSI-based counting does not allow. It also never applied California's $230/month income disregard, which brings countable income to $21,093.47 ($1,757.79/month). That is under the actual 138% FPL limit of $22,024.80 ($1,835.40/month), not over its ~$1,800 estimate."
-us,scenario_031,head_medicaid_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"It applied SSI's $20/month ($240/year) general exclusion instead of California's $230/month ($2,760/year) disregard, which replaces it. It also understated the 2026 limit as ~$21,600 instead of $22,024.80. With the correct disregard, countable income is $21,093.47, which is eligible."
-us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"It used a badly understated 138% FPL limit (~$18,564 instead of $22,024.80), compared gross income with no California $230/month disregard, and suggested $4,200 in assets could exceed the resource cap. California's individual asset limit is $130,000, and countable income after the $2,760 disregard is $21,093.47, which is eligible."
-us,scenario_031,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It treated Medicare age as ruling out Medicaid and never ran the senior-or-disabled pathway. Age 67 meets that category on its own, and Medicare eligibility bars only the adult expansion group. Countable income of $21,093.47 after California's $230/month disregard is under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a 100% FPL limit (~$15,650) for the aged pathway when California's senior-or-disabled limit is 138% of the 2026 FPL ($22,024.80). It also skipped the $230/month state disregard that brings countable income to $21,093.47."
-us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,state_local_rule,False,"It compared gross income of $23,853 to the aged limit without California's $230/month ($2,760/year) disregard, and used an outdated limit of ~$21,600. Countable income is $21,093.47, under the 2026 limit of 138% FPL ($22,024.80)."
-us,scenario_031,head_medicaid_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"It compared gross income of $23,853 to an understated 138% FPL figure (~$21,600) and never subtracted California's $2,760 annual disregard. Countable income of $21,093.47 is under the 2026 limit of $22,024.80."
-us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"It went back and forth between MAGI expansion rules (which don't apply at 67) and a 100% FPL limit, then compared gross income to a stale 138% FPL figure of $20,783. It never applied California's $230/month disregard in the senior-or-disabled income test, which yields countable income of $21,093.47 against a $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It excluded most Social Security, but SSI-based counting includes it in full. It then applied an SSI federal benefit rate limit (~$13,596) instead of California's 138% FPL limit for senior-or-disabled Medi-Cal ($22,024.80). The correct countable income is $21,093.47 after the $2,760 state disregard, which is eligible."
-us,scenario_031,head_medicaid_eligible,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It said the head is not in a categorical group, but age 67 alone places them in the aged senior-or-disabled category. It also compared gross income to 138% FPL without California's $230/month disregard, which leaves $21,093.47, under $22,024.80."
-us,scenario_031,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It compared gross income of $23,853 to 138% FPL and missed California's $230/month ($2,760/year) disregard. That disregard brings countable income to $21,093.47, under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"It compared gross monthly income of $1,987.75 to an understated $1,732 limit. The 2026 138% FPL limit is $1,835.40/month, and after California's $230/month disregard countable income is $1,757.79, so the head is eligible."
-us,scenario_031,head_medicaid_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It treated the lack of SSI as relevant and compared gross income of $23,853 to the FPL limit. Aged status meets the category without SSI, and California's $2,760 annual disregard brings countable income to $21,093.47, under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It compared gross income of $23,853 to a ~$22,218 limit without California's $230/month disregard. Subtracting $2,760 gives countable income of $21,093.47, under the 2026 138% FPL limit of $22,024.80."
-us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"It compared gross income of $23,853 to ~$22,080 and skipped California's $230/month income disregard for the aged and disabled program. Countable income of $21,093.47 is under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It applied a generic single-adult income limit to gross income instead of the aged senior-or-disabled test. That test subtracts California's $2,760 annual disregard, which leaves $21,093.47, under the 138% FPL limit of $22,024.80."
-us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It compared gross income of $23,853 directly to 138% FPL and omitted California's $230/month disregard. With the disregard, countable income is $21,093.47, under $22,024.80."
-us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,"It compared gross monthly income (~$1,988) to the 138% FPL non-MAGI threshold without subtracting California's $230/month disregard. That disregard leaves $1,757.79/month, under the $1,835.40 monthly limit."
-us,scenario_031,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It rejected eligibility based on age and income without running the senior-or-disabled income test. That test subtracts California's $2,760 annual disregard, which yields countable income of $21,093.47, under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"It compared gross annual income of $23,853 to the 65+ threshold and omitted California's $230/month disregard. With it, countable income is $21,093.47, under 138% of the 2026 FPL ($22,024.80)."
-us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"It judged income over the non-MAGI limit without subtracting California's $230/month ($2,760/year) disregard. That brings countable income from $23,853.47 to $21,093.47, under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"It compared total income to 138% FPL with no deductions and missed California's $2,760 annual disregard. That disregard leaves countable income of $21,093.47, under $22,024.80."
-us,scenario_031,head_medicaid_eligible,gemini-4-argon,llm_error,state_local_rule,False,"It treated gross monthly income of $1,987.75 as countable and missed California's $230/month disregard, which reduces it to $1,757.79. That is under the actual 2026 138% FPL monthly limit of $1,835.40, not over its ~$1,800 estimate."
-us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"It assumed Medicare eligibility at 65+ rules out Medicaid. In the engine, Medicare bars only the 19–64 adult expansion group, and the aged senior-or-disabled pathway remains open and passes, with $21,093.47 countable against a $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"It tested the head against the adult expansion group, which ends at 64, instead of the aged senior-or-disabled category. That category counts income after California's $230/month disregard: $21,093.47, under the 138% FPL limit of $22,024.80."
-us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It required a disability or other qualifying flag, but age 67 alone satisfies the aged condition of the senior-or-disabled category. Countable income after California's $2,760 disregard is $21,093.47, under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,state_local_rule,False,"It applied a medically-needy/spend-down standard instead of California's senior-or-disabled pathway at 138% FPL. Under that pathway, the $230/month disregard leaves countable income of $21,093.47, under $22,024.80, so no spend-down is needed."
-us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"It judged countable income over the threshold without subtracting California's $230/month disregard from the $23,853.47 total. The disregard yields $21,093.47, under 138% of the 2026 FPL ($22,024.80)."
-us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,"It compared retirement income to the aged limit without California's $2,760 annual disregard. Countable income of $21,093.47 is under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It tested income against an SSI-linked income level. California's senior-or-disabled pathway uses 138% of FPL ($22,024.80) with a $230/month disregard, which gives countable income of $21,093.47."
-us,scenario_031,head_medicaid_eligible,gpt-6-astra,llm_error,state_local_rule,False,"It said retirement income exceeded California's aged limit after the general income exclusion, without stating the exclusion's amount. In the engine that exclusion is California's $230/month disregard, which replaces SSI's $20/month general income exclusion. Using the $230 disregard brings countable income to $21,093.47, under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,gpt-6-sol,llm_error,state_local_rule,False,"It compared retirement income to the aged threshold without California's $230/month disregard. With the disregard, countable income is $21,093.47, under 138% FPL ($22,024.80)."
-us,scenario_031,head_medicaid_eligible,gpt-6.1-sol,llm_error,state_local_rule,False,"It said retirement income exceeded California's aged limit after the applicable general income exclusion, without stating the exclusion's amount. The engine applies California's $230/month disregard. The correct disregard brings countable income to $21,093.47, under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,gpt-6.1-terra,llm_error,state_local_rule,False,"It compared annual retirement income to the aged limit without California's $2,760 annual disregard. Countable income is $21,093.47, under $22,024.80."
-us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It assumed PolicyEngine excludes Medicare-eligible people from Medicaid. Medicare eligibility bars only the adult expansion group, and the aged senior-or-disabled pathway passes, with $21,093.47 countable against a $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"It claimed $4,200 in bank assets exceeds California's aged resource limit, which is $130,000 for an individual. It also compared gross income of $23,853 to the limit without the $2,760 annual state disregard, which leaves $21,093.47, under $22,024.80."
-us,scenario_031,head_medicaid_eligible,grok-4.6,llm_error,state_local_rule,False,"It treated $23,853 as countable income and skipped California's $230/month disregard. With it, income falls to $21,093.47, under the 138% FPL Aged & Disabled limit of $22,024.80."
-us,scenario_031,head_medicaid_eligible,grok-4.7,llm_error,state_local_rule,False,"It applied SSI's $20/month unearned-income disregard instead of California's $230/month disregard, which replaces it. The correct disregard brings countable income to $21,093.47, under the 138% FPL limit of $22,024.80."
-us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"It applied the $2,000 SSI resource limit, but California's senior-or-disabled asset limit is $130,000 for an individual, so $4,200 passes easily. Income also passes: $21,093.47 after the $230/month disregard, under $22,024.80."
-us,scenario_031,head_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used a 100% FPL limit (~$15,650) for aged Medi-Cal instead of 138% of the 2026 FPL ($22,024.80). It also compared gross income with no $230/month California disregard, which brings countable income to $21,093.47."
-us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"It required disability, blindness, or SSI receipt for the non-MAGI pathway. Age 67 alone makes the head aged and satisfies is_ssi_aged_blind_disabled. Countable income of $21,093.47 after California's $230/month disregard is under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,state_local_rule,False,"It compared retirement income to the California aged limit without subtracting the $2,760 annual state disregard. That leaves countable income of $21,093.47, under 138% FPL ($22,024.80)."
-us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,state_local_rule,False,"It guessed that roughly $24k of income exceeds the senior limit without calculating anything. The aged test subtracts California's $2,760 disregard from $23,853.47, which leaves $21,093.47, under the $22,024.80 limit."
-us,scenario_031,head_medicaid_eligible,muse-spark-1.3,llm_error,categorical_eligibility,False,"It applied a MAGI test with an alimony deduction, which belongs to the under-65 expansion group, instead of the aged senior-or-disabled test with SSI-based counting. That test subtracts California's $230/month disregard and gives $21,093.47, under $22,024.80."
-us,scenario_031,head_medicaid_eligible,muse-wonder,llm_error,state_local_rule,False,"It applied SSI's $20/month disregard (giving ~$1,967/month) instead of California's $230/month disregard. The correct disregard leaves $1,757.79/month, under the $1,835.40 monthly limit at 138% FPL."
-us,scenario_031,head_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"It applied the $2,000 SSI resource limit, but California's senior-or-disabled asset limit is $130,000 for an individual. It also compared ~$24,000 gross income to the limit without the $2,760 annual state disregard, which leaves $21,093.47, under $22,024.80."
-us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"It treated Medicare eligibility and the lack of SSI or disability status as ruling out Medicaid. Age 67 alone qualifies for the senior-or-disabled category, and countable income after California's $230/month disregard is $21,093.47, under $22,024.80."
-us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It said the head is in no eligibility category, but being aged (67) is itself the senior-or-disabled category. Countable income of $21,093.47 after California's $2,760 disregard is under the 138% FPL limit of $22,024.80."
-us,scenario_031,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income test to this elderly household and denied benefits. California's BBCE (gross income under 200% FPL) makes the household categorically eligible, which waives the net income test. As a one-person household it then gets the $24/month minimum allotment, or $288 a year."
-us,scenario_031,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"The model treated the 100% FPL net income test as binding and stopped once 30% of net income exceeded $298. Under California BBCE, gross income of about $1,988/month (under 200% FPL) confers categorical eligibility, which waives the net test. The one-person household therefore receives the $24 monthly minimum allotment, or $288 a year."
-us,scenario_031,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied a 130% FPL gross income test, which elderly households are exempt from anyway. It also missed that California's BBCE uses a 200% FPL gross limit (about $2,608/month), which this household meets. Categorical eligibility then entitles the one-person household to the $24/month minimum benefit ($288 a year)."
-us,scenario_031,snap,claude-haiku-5.5,llm_error,asset_resource,False,"The model denied SNAP because the $4,200 in bank assets supposedly exceeded the resource limit. Under California's BBCE there is no asset test, and the elderly-household limit is higher than $4,200 anyway. The household is categorically eligible and receives the $24/month minimum allotment ($288 a year)."
-us,scenario_031,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model said elderly households must pass the 100% FPL net income test and found net income of about $1,778 over the limit. California BBCE (gross income under 200% FPL) waives the net income test. As a categorically eligible one-person household, it gets the $24/month minimum allotment, or $288 a year."
-us,scenario_031,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model denied benefits because net income was above the 100% FPL net limit. California's 200% FPL BBCE pathway makes the household categorically eligible and waives that test. The one-person minimum allotment of $24/month then applies, for $288 a year."
-us,scenario_031,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model claimed income exceeds CalFresh's gross and net limits. Gross income of $1,988/month is under California's 200% FPL BBCE limit of about $2,608, which confers categorical eligibility and waives the net test. That makes the household eligible for the $24/month minimum allotment ($288 a year)."
-us,scenario_031,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the $298 maximum allotment, but then set the benefit to $0. A one-person household that is categorically eligible under California BBCE receives the minimum allotment of $24/month (8% of the one-person maximum), or $288 a year."
-us,scenario_031,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model applied a 130% FPL gross income test, but elderly households are exempt from it. It also ignored California's BBCE 200% FPL gross limit, which the $1,988/month income meets. Categorical eligibility then gives the one-person household the $24/month minimum benefit, or $288 a year."
-us,scenario_031,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model used a gross limit of about $1,632/month and a 100% FPL net limit. California's BBCE gross limit is 200% FPL (about $2,608/month), which this household meets, and BBCE waives the net test. The categorically eligible one-person household receives the $24/month minimum allotment ($288 a year)."
-us,scenario_031,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"The model stopped at a computed benefit of $0 because 30% of net income exceeded the maximum allotment. It never applied the minimum benefit rule (7 U.S.C. 2017(a)), which pays categorically eligible one-person households $24/month, or $288 a year."
-us,scenario_031,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"The model computed a negative formula benefit ($292 minus $535) and reported $0. Under California BBCE the household is categorically eligible, so the one-person minimum allotment of $24/month applies instead, for $288 a year."
-us,scenario_031,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model denied benefits because net income of $1,783.75 exceeded the 100% FPL net limit. California's 200% FPL BBCE pathway makes the household categorically eligible and waives the net test. The one-person minimum allotment of $24/month then yields $288 a year."
-us,scenario_031,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"The model applied the 100% FPL net income test and found the household ineligible. California BBCE (gross income under 200% FPL) waives that test, making the household categorically eligible for the $24/month one-person minimum benefit, or $288 a year."
-us,scenario_031,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"The model treated net income of about $1,790 above 100% FPL as disqualifying. Under California's BBCE, gross income under 200% FPL confers categorical eligibility and waives the net test. The one-person household gets the $24/month minimum allotment ($288 a year)."
-us,scenario_031,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model incorrectly said California BBCE removes the gross income limit; BBCE uses a 200% FPL gross limit. It then applied the 100% FPL net test and returned $0. Categorical eligibility waives the net test, and one-person categorically eligible households receive the $24/month minimum allotment, or $288 a year."
-us,scenario_031,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model asserted that income exceeds the SNAP threshold without applying California's 200% FPL BBCE limit, which the $1,988/month gross income meets. Categorical eligibility gives the one-person household the $24/month minimum benefit, or $288 a year."
-us,scenario_031,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly used BBCE categorical eligibility and the minimum benefit, but applied the FY2025 minimum of $23. The FY2026 minimum allotment is $24/month (8% of the $298 one-person maximum), which gives $288 a year, not $276."
-us,scenario_031,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly applied BBCE categorical eligibility and the one-person minimum benefit, but used the outdated $23 monthly minimum. The FY2026 minimum is $24/month (8% of the $298 maximum allotment), which yields $288 a year."
-us,scenario_031,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model claimed income and assets exceed SNAP thresholds. Under California BBCE, gross income under 200% FPL confers categorical eligibility with no asset test. The one-person household receives the $24/month minimum allotment, or $288 a year."
-us,scenario_031,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model applied the net income limit and the zero formula benefit, then stopped at $0. BBCE categorical eligibility waives the net test, and the one-person minimum allotment of $24/month applies, for $288 a year."
-us,scenario_031,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model reported $0 because the 30% net income contribution exceeds the maximum allotment. It missed that categorically eligible one-person households receive the $24/month minimum allotment regardless, which totals $288 a year."
-us,scenario_031,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"The model treated the net income threshold as disqualifying. California's BBCE (gross income under 200% FPL) waives the net test, and the formula's zero result is replaced by the $24/month one-person minimum benefit, or $288 a year."
-us,scenario_031,snap,gemini-4-argon,llm_error,categorical_eligibility,False,"The model considered BBCE but wrongly concluded that the minimum benefit does not apply when 30% of net income exceeds the maximum allotment. Categorically eligible one- and two-person households receive the $24/month minimum allotment whenever the computed benefit falls below it, including at $0, which gives $288 a year."
-us,scenario_031,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model correctly found the household meets California's 200% FPL BBCE limit, but then reported the formula's $0 as the benefit. A categorically eligible one-person household receives the $24/month minimum allotment, or $288 a year."
-us,scenario_031,snap,glm-5.3,llm_error,categorical_eligibility,False,"The model correctly passed the household on California's 200% FPL gross test but then applied the 100% FPL net income test. BBCE categorical eligibility waives the net test. The household gets the $24/month one-person minimum allotment, or $288 a year."
-us,scenario_031,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the household as having very low countable income and produced a regular benefit of $144/month. Countable income is about $1,988/month (Social Security, pension, and IRA), so 30% of net income exceeds the $298 maximum. The household receives only the $24/month minimum allotment ($288 a year)."
-us,scenario_031,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model overstated income at about $31k (the actual total is $23,853) and applied generic thresholds without California's 200% FPL BBCE test. That test makes the household categorically eligible for the $24/month one-person minimum benefit, or $288 a year."
-us,scenario_031,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model stopped at a $0 computed allotment after deductions. It never applied the minimum allotment rule, under which a categorically eligible (BBCE) one-person household receives $24/month, or $288 a year."
-us,scenario_031,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model denied benefits because net income stayed above the net income limit. It also treated alimony paid as a deduction, though SNAP does not deduct alimony. California BBCE waives the net test, so the household receives the $24/month one-person minimum allotment, or $288 a year."
-us,scenario_031,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model applied the net income limit to this elderly household without recognizing that California's 200% FPL BBCE pathway confers categorical eligibility and waives the net test. The one-person minimum allotment of $24/month then applies, for $288 a year."
-us,scenario_031,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"The model reported the formula's $0 result. A categorically eligible one-person household receives the $24/month minimum allotment (7 U.S.C. 2017(a)) when the computed benefit falls below it, for $288 a year."
-us,scenario_031,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"The model concluded that no allotment remains after deductions and returned $0. It missed the minimum benefit, which guarantees a categorically eligible one-person household $24/month, or $288 a year."
-us,scenario_031,snap,gpt-6.1-terra,llm_error,categorical_eligibility,False,"The model treated income that is too high for a positive formula benefit as meaning $0. Under California BBCE the household is categorically eligible, so the $24/month one-person minimum allotment applies, for $288 a year."
-us,scenario_031,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model applied the 130% FPL gross test, which elderly households are exempt from, and also claimed the assets exceed the limit. California BBCE (200% FPL gross, no asset test) makes the household categorically eligible for the $24/month minimum benefit, or $288 a year."
-us,scenario_031,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model failed the household on the 100% FPL net income test. California BBCE categorical eligibility (gross income under 200% FPL) waives that test, and the one-person household receives the $24/month minimum allotment, or $288 a year."
-us,scenario_031,snap,grok-4.6,llm_error,categorical_eligibility,False,"The model found that the 30% benefit reduction exceeds the maximum allotment and reported $0. It never applied the minimum allotment, which gives a categorically eligible one-person household $24/month, or $288 a year."
-us,scenario_031,snap,grok-4.7,llm_error,categorical_eligibility,False,"The model applied the net income limit, and it deducted alimony, which SNAP does not allow. California BBCE waives the net test, so the categorically eligible one-person household receives the $24/month minimum allotment, or $288 a year."
-us,scenario_031,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model compared annual net income to a net income limit of about $16,000 and denied benefits. California BBCE categorical eligibility waives the net test, and the one-person household gets the $24/month minimum allotment, or $288 a year."
-us,scenario_031,snap,inkling,llm_error,categorical_eligibility,False,"The model correctly waived the gross test for an elderly household but then applied the 100% FPL net test. California BBCE (gross income under 200% FPL) waives the net test too. The household receives the $24/month one-person minimum benefit, or $288 a year."
-us,scenario_031,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model found the household ineligible because income exceeds the net income limit. California's BBCE pathway confers categorical eligibility at under 200% FPL gross income and waives the net test, which yields the $24/month one-person minimum allotment, or $288 a year."
-us,scenario_031,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model correctly found no shelter deduction but omitted the medical deduction for modeled Medicare Part B premiums and applied the one-person net income limit. BBCE categorical eligibility waives the net test, and the one-person minimum allotment of $24/month applies, for $288 a year."
-us,scenario_031,snap,minimax-m3,llm_error,asset_resource,False,"The model claimed $4,200 in bank assets exceeds a $3,500 elderly resource limit. California BBCE imposes no asset test, and the federal elderly limit is above $4,200 anyway. The categorically eligible household receives the $24/month minimum allotment, or $288 a year."
-us,scenario_031,snap,muse-spark-1.3,llm_error,categorical_eligibility,False,"The model ruled the household ineligible because net income exceeds the 100% FPL limit. California BBCE waives the net test for households under 200% FPL gross income. The one-person household gets the $24/month minimum benefit, or $288 a year."
-us,scenario_031,snap,muse-wonder,llm_error,categorical_eligibility,False,"The model found net income of about $1,779 above 100% FPL and denied benefits. California BBCE categorical eligibility waives the net income test, and the one-person minimum allotment of $24/month applies, for $288 a year."
-us,scenario_031,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model correctly computed the formula benefit ($298 minus about $535) as negative but reported $0. It did not apply the minimum allotment, which pays a categorically eligible one-person household $24/month, or $288 a year."
-us,scenario_031,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model applied a 130% FPL gross test and failed the household on it. Elderly households are exempt from that test, and California's BBCE 200% FPL limit (about $2,608/month) is met. The household is categorically eligible for the $24/month minimum benefit, or $288 a year."
-us,scenario_031,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model miscounted annual countable income as $19,765.50 and then treated it as above California's limit. Countable gross income of about $1,988/month is under the 200% FPL BBCE limit, so the household is categorically eligible for the $24/month one-person minimum allotment, or $288 a year."
+us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,state_local_rule,False,"It subtracted the $1,165 alimony expense, which SSI-based counting does not allow, and never applied California's $230/month ($2,760/yr) disregard. It also understated the 2026 138% FPL limit, which is $22,024.80 ($1,835.40/month). The correct countable income is $21,093.47, which is under the limit."
+us,scenario_031,head_medicaid_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"It applied SSI's $20/month ($240/yr) general exclusion, but California's aged-and-disabled program replaces it with a $230/month ($2,760/yr) disregard. It also used about $21,600 instead of the 2026 limit of $22,024.80. With the correct disregard, countable income is $21,093.47, which is under the limit."
+us,scenario_031,head_medicaid_eligible,claude-fable-5.2,llm_error,state_local_rule,False,"It reduced $23,853 by only the $240 SSI general exclusion and missed California's $230/month disregard, which removes $2,760 a year. That leaves countable income of $21,093.47, below the $22,024.80 limit (138% FPL)."
+us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"It took 138% FPL as about $18,564 when the 2026 figure is $22,024.80, compared gross income with no disregard, and suggested the $4,200 in assets could exceed the resource limit. California's $230/month disregard brings countable income to $21,093.47, and the 2026 individual asset limit is $130,000."
+us,scenario_031,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It treated being Medicare-age as ruling out Medicaid. Medicare eligibility bars only the 19–64 expansion group. Age 67 alone qualifies the head for California's aged-and-disabled pathway, where countable income of $21,093.47 after the $230/month disregard is under $22,024.80 and assets are under $130,000."
+us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a 100% FPL limit (about $15,650) and gross income with no disregard. California's aged-and-disabled pathway uses 138% FPL ($22,024.80 in 2026) and a $230/month disregard, which gives countable income of $21,093.47."
+us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,state_local_rule,False,"It compared gross income of $23,853 with a 100–138% FPL range and applied no disregard. California subtracts a $230/month ($2,760/yr) disregard and uses 138% FPL ($22,024.80), so countable income of $21,093.47 qualifies."
+us,scenario_031,head_medicaid_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"It compared gross income of $23,853 with about $21,600, skipping California's $230/month disregard and understating the 2026 limit of $22,024.80. After the $2,760 disregard, income is $21,093.47, which is under the limit."
+us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"It switched between MAGI rules, a 100% FPL test and a wrong 138% FPL figure of $20,783, and settled on the ACA expansion test, which does not apply at 67. The correct test is the non-MAGI aged-and-disabled pathway: $23,853.47 less the $2,760 California disregard gives $21,093.47, which is under $22,024.80."
+us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It wrongly left most Social Security out of countable income, then tested the result against the SSI federal benefit rate (about $13,596). California's aged-and-disabled limit is 138% FPL ($22,024.80), and full income less the $2,760 disregard comes to $21,093.47, which is under it."
+us,scenario_031,head_medicaid_eligible,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It said the head is in no categorical group, but age 67 is enough for the aged pathway. It then compared gross income of about $23.9k with 138% FPL without California's $230/month disregard, which lowers countable income to $21,093.47, below $22,024.80."
+us,scenario_031,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It compared gross income of $23,853 with 138% FPL and never subtracted California's $230/month ($2,760/yr) aged-and-disabled disregard. That disregard brings countable income to $21,093.47, under the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"It tested gross monthly income of $1,987.75 against an underestimated limit of $1,732. The 2026 limit is $1,835.40/month. After California's $230/month disregard, countable income is $1,757.79/month, which is under the limit."
+us,scenario_031,head_medicaid_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It treated the lack of SSI as relevant, though age alone qualifies the head categorically. It also compared gross income of $23,853 with the limit without California's $230/month disregard, which leaves $21,093.47 against the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It compared gross income of $23,853 with a limit of about $22,218 and applied no disregard. Subtracting California's $2,760 annual disregard gives $21,093.47, which is under the 2026 limit of $22,024.80."
+us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"It correctly used the 138% FPL limit (about $22,080) but compared gross income of $23,853 to it. It omitted California's $230/month disregard, which reduces countable income to $21,093.47, below the limit."
+us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It asserted that income exceeds the limit without applying California's aged-and-disabled income counting. Counted income of $23,853.47 less the $2,760 annual disregard is $21,093.47, which is under the 138% FPL limit of $22,024.80."
+us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It tested gross income of $23,853 against 138% FPL and missed California's $230/month disregard. That disregard brings countable income to $21,093.47, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,"It compared gross monthly income (about $1,988) with the 138% FPL non-MAGI limit and skipped the $230/month California disregard. That disregard reduces countable income to $1,757.79/month, under the $1,835.40 limit."
+us,scenario_031,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It concluded ineligibility without applying the aged-and-disabled income test. Income of $23,853.47 less California's $2,760 disregard is $21,093.47, which is under the 138% FPL limit of $22,024.80, and $4,200 in assets is under $130,000."
+us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"It compared gross income of $23,853 with the aged threshold and never subtracted California's $230/month disregard. With the disregard, countable income is $21,093.47, under the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"It judged income to exceed the non-MAGI limit without subtracting California's $230/month ($2,760/yr) disregard. That leaves countable income of $21,093.47, under the 2026 limit of 138% FPL ($22,024.80)."
+us,scenario_031,head_medicaid_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"It compared total income with 138% FPL using gross figures. California's aged-and-disabled program subtracts a $2,760 annual disregard, which leaves $21,093.47, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,gemini-4-argon,llm_error,state_local_rule,False,"It took gross income of $1,987.75/month as countable and skipped California's $230/month disregard. That disregard leaves $1,757.79/month, which is under the 2026 limit of $1,835.40/month (138% FPL)."
+us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"It assumed Medicare eligibility rules out Medicaid. Medicare bars only the 19–64 expansion group, and the head qualifies through the aged-and-disabled pathway: $21,093.47 countable income after the $230/month disregard is under $22,024.80, and assets are under $130,000."
+us,scenario_031,head_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"It applied the ACA expansion test, which does not apply at 67, and used a wrong FPL figure, putting income at 199% FPL when gross income is 149% FPL. The correct test is the aged-and-disabled pathway, where income less the $230/month disregard comes to $21,093.47, under the 138% FPL limit of $22,024.80."
+us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It required a disability or other qualifying flag, but age 67 alone qualifies the head for the aged-and-disabled pathway. It also never applied California's $2,760 disregard, which brings countable income to $21,093.47, under the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,state_local_rule,False,"It tested the head against a medically needy (spend-down) standard instead of California's 138% FPL aged-and-disabled program. Under that program, income less the $230/month disregard is $21,093.47, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"It asserted that income exceeds the threshold without subtracting California's $230/month aged-and-disabled disregard. That disregard reduces countable income from $23,853.47 to $21,093.47, below the 138% FPL limit of $22,024.80."
+us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,"It compared retirement income with the aged limit without California's $2,760 annual disregard. That disregard leaves $21,093.47 in countable income, under the 2026 limit of 138% FPL ($22,024.80)."
+us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It tested income against an SSI-linked level instead of California's aged-and-disabled limit of 138% FPL ($22,024.80). Under that limit, income less the $230/month disregard ($21,093.47) qualifies."
+us,scenario_031,head_medicaid_eligible,gpt-6-astra,llm_error,state_local_rule,False,"It applied SSI's $20/month general income exclusion, which California replaces with a $230/month disregard for the aged-and-disabled program. With the $2,760 annual disregard, countable income is $21,093.47, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,gpt-6-sol,llm_error,state_local_rule,False,"It judged retirement income too high without subtracting California's $230/month disregard. That disregard reduces countable income to $21,093.47, under the 138% FPL limit of $22,024.80."
+us,scenario_031,head_medicaid_eligible,gpt-6.1-sol,llm_error,state_local_rule,False,"It used SSI's $20 general income exclusion, but California substitutes a $230/month ($2,760/yr) disregard. With that disregard, countable income is $21,093.47, under the $22,024.80 limit."
+us,scenario_031,head_medicaid_eligible,gpt-6.1-terra,llm_error,state_local_rule,False,"It ran the aged income test on retirement income without California's $2,760 annual disregard. Countable income of $21,093.47 is under the 2026 limit of 138% FPL ($22,024.80)."
+us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It assumed Medicare eligibility excludes Medicaid. PolicyEngine applies the Medicare bar only to the 19–64 expansion group, and the head qualifies through the aged-and-disabled pathway with $21,093.47 countable income, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"It claimed $4,200 in assets exceeds the aged resource limit, but California's 2026 limit for an individual is $130,000. It also compared gross income of $23,853 without the $2,760 disregard, which brings countable income to $21,093.47, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,grok-4.6,llm_error,state_local_rule,False,"It counted $23,853 against the aged-and-disabled limit without subtracting California's $230/month disregard. That disregard leaves $21,093.47, under the 138% FPL limit of $22,024.80."
+us,scenario_031,head_medicaid_eligible,grok-4.7,llm_error,state_local_rule,False,"It applied SSI's $20/month unearned-income disregard instead of California's $230/month aged-and-disabled disregard. The correct disregard brings countable income to $21,093.47, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"It applied the $2,000 SSI resource limit. California's 2026 aged-and-disabled asset limit for an individual is $130,000, so $4,200 passes easily, and income after the $230/month disregard is under 138% FPL."
+us,scenario_031,head_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used a 100% FPL limit (about $15,650) and gross income. California's aged-and-disabled program uses 138% FPL ($22,024.80) and a $230/month disregard, which gives countable income of $21,093.47."
+us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"It held that the non-MAGI aged pathway requires disability, blindness or SSI receipt, but being 65 or older alone meets the aged category. Countable income of $21,093.47 after the $230/month disregard is then under $22,024.80."
+us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,state_local_rule,False,"It judged retirement income above the aged limit without subtracting California's $2,760 annual disregard. With it, countable income is $21,093.47, under the 138% FPL limit of $22,024.80."
+us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,state_local_rule,False,"It guessed that income of about $24k exceeds the limit without computing countable income. California's $230/month disregard reduces income to $21,093.47, below the 138% FPL limit of $22,024.80."
+us,scenario_031,head_medicaid_eligible,muse-spark-1.3,llm_error,state_local_rule,False,"It applied MAGI rules with alimony subtracted ($22,688), but a 67-year-old is tested under non-MAGI SSI-based counting. That counting does not subtract alimony and does apply California's $2,760 disregard, giving $21,093.47, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,muse-wonder,llm_error,state_local_rule,False,"It applied SSI's $20/month disregard, getting about $1,967/month, instead of California's $230/month disregard. The correct disregard gives $1,757.79/month, under the 2026 limit of $1,835.40/month (138% FPL)."
+us,scenario_031,head_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"It applied the $2,000 SSI resource limit, but California's 2026 limit for an individual is $130,000. It also compared gross income without the $230/month disregard, which brings countable income to $21,093.47, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"It required disability, nursing-home or SSI status, but age 67 alone qualifies the head for the aged pathway. It also compared gross income of $23,853 without California's $2,760 disregard, which leaves $21,093.47, under $22,024.80."
+us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It said the head fits no eligibility category, but being 65 or older is itself the aged category. Under that category, income of $23,853.47 less the $2,760 California disregard is under the 138% FPL limit of $22,024.80."
+us,scenario_031,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Held the household to the 100% FPL net income test. But its $1,988 gross is under California's 200% FPL broad-based categorical eligibility limit ($2,608), which waives the net test. The eligible one-person household gets the $24/month minimum allotment ($288/yr), not $0."
+us,scenario_031,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Said the household fails the 100% FPL net income test and zeroed the benefit. It never applied California's 200% FPL broad-based categorical eligibility, which waives the net test. It also missed that an eligible one-person household receives the $24/month minimum allotment even when 30% of net income exceeds $298."
+us,scenario_031,snap,claude-fable-5.2,llm_error,thresholds_rates,False,"Correctly found categorical eligibility and the minimum allotment, but used 8% of $298 unrounded ($23.84). The minimum allotment is rounded to the nearest whole dollar, $24/month, so the annual amount is $288, not $286.08."
+us,scenario_031,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Applied a 130% FPL gross income test. That test does not bind an elderly household, and in California the 200% FPL broad-based categorical eligibility limit applies. At $1,988/month the household is categorically eligible and receives the $24/month minimum allotment ($288/yr)."
+us,scenario_031,snap,claude-haiku-5.5,llm_error,asset_resource,False,"Said $4,200 in bank assets exceeds the elderly resource limit. It does not: the federal elderly limit is $4,500, and California's broad-based categorical eligibility waives the asset test anyway. The household is eligible and gets the $24/month minimum allotment."
+us,scenario_031,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"Treated the 100% FPL net income test as disqualifying. But gross income of $1,988 is under California's 200% FPL broad-based categorical eligibility limit, which waives that test. The categorically eligible one-person household gets the $24/month minimum allotment ($288/yr)."
+us,scenario_031,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Disqualified the household on the net income limit (using the outdated $1,255). California's 200% FPL broad-based categorical eligibility, which this household meets at $1,988/month, removes the net income test. It also missed the $24/month minimum allotment for eligible one-person households."
+us,scenario_031,snap,claude-opus-5,llm_error,categorical_eligibility,False,"Said income exceeds the CalFresh gross and net limits. In fact $1,988/month is under California's 200% FPL broad-based categorical eligibility gross limit ($2,608), and categorical eligibility waives the net test. The household receives the $24/month minimum allotment."
+us,scenario_031,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"Correctly computed that 30% of net income exceeds the $298 maximum but stopped at $0. A one-person household that is categorically eligible under California's 200% FPL broad-based categorical eligibility receives the $24/month minimum allotment, $288/yr."
+us,scenario_031,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Applied a 130% FPL gross income test (understated at $1,580). Elderly households are exempt from that test, and California's broad-based categorical eligibility sets a 200% FPL ($2,608) limit that this household meets. The result is categorical eligibility and the $24/month minimum allotment."
+us,scenario_031,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"Used a roughly $1,632 gross limit and the $1,255 net limit to deny eligibility. California's broad-based categorical eligibility gross limit is 200% FPL ($2,608), which the $1,988 income passes, and that waives the net test. The eligible household gets the $24/month minimum allotment."
+us,scenario_031,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Computed that 30% of net income exceeds the one-person maximum and returned $0. It missed that a categorically eligible one-person household gets the $24/month minimum allotment under 7 U.S.C. 2017(a), which is $288/yr."
+us,scenario_031,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Zeroed the benefit because 30% of net income exceeds the maximum allotment. It ignored that the household is categorically eligible under California's 200% FPL broad-based categorical eligibility, so the $24/month minimum allotment applies. It also used the outdated $292 maximum instead of $298."
+us,scenario_031,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Disqualified the household on the 100% FPL net income limit. California's 200% FPL broad-based categorical eligibility, which the $1,988/month gross meets, waives the net test. The one-person household gets the $24/month minimum allotment ($288/yr)."
+us,scenario_031,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Applied the 100% FPL net income test as an eligibility bar. Under California's 200% FPL broad-based categorical eligibility that test is waived, and the eligible one-person household receives the $24/month minimum allotment."
+us,scenario_031,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Disqualified the household because net income of about $1,790 exceeds 100% FPL. California's broad-based categorical eligibility (gross under 200% FPL) waives the net test, so the household gets the $24/month minimum allotment ($288/yr)."
+us,scenario_031,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"Recognized that broad-based categorical eligibility removes the gross limit, but still applied the 100% FPL net income test. It also returned $0 for the negative formula result. Categorical eligibility waives the net test, and eligible one-person households get the $24/month minimum allotment."
+us,scenario_031,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Asserted that income exceeds the SNAP threshold without applying California's 200% FPL broad-based categorical eligibility limit, which $1,988/month passes. The categorically eligible one-person household receives the $24/month minimum allotment ($288/yr)."
+us,scenario_031,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Correctly applied broad-based categorical eligibility and the minimum allotment, but used the outdated $23 minimum (8% of $292). For 2026 the minimum is 8% of the $298 one-person maximum, rounded to $24/month, giving $288, not $276."
+us,scenario_031,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"Correctly applied categorical eligibility and the minimum benefit, but used the prior-year $23/month minimum. The FY2026 minimum is 8% of $298, rounded to $24, which gives $288/yr instead of $276."
+us,scenario_031,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Claimed income and assets exceed SNAP thresholds. In fact $1,988/month is under California's 200% FPL broad-based categorical eligibility limit, and $4,200 of assets passes. The eligible one-person household gets the $24/month minimum allotment."
+us,scenario_031,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"Zeroed the benefit using the net income limit and the benefit formula. California's broad-based categorical eligibility waives the net test, and an eligible one-person household gets the $24/month minimum allotment even when the formula is negative."
+us,scenario_031,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,Correctly found that 30% of net income exceeds the maximum allotment but returned $0. It omitted the $24/month minimum allotment that categorically eligible one- and two-person households receive under 7 U.S.C. 2017(a).
+us,scenario_031,snap,gemini-3.8-flash,llm_error,categorical_eligibility,False,"Treated the net income threshold as disqualifying and returned $0. California's 200% FPL broad-based categorical eligibility waives the net test, and the eligible one-person household receives the $24/month minimum allotment ($288/yr)."
+us,scenario_031,snap,gemini-4-argon,llm_error,categorical_eligibility,False,"Held that the minimum benefit does not apply at this income level. The minimum allotment applies to every eligible one- or two-person household however negative the formula result is, and broad-based categorical eligibility makes this household eligible. It also used the outdated $292 maximum and $23 minimum; the correct minimum is $24/month."
+us,scenario_031,snap,glm-5.2,llm_error,categorical_eligibility,False,"Correctly found the household eligible under California's 200% FPL broad-based categorical eligibility, then returned $0 because 30% of net income exceeds the maximum. It missed that an eligible one-person household receives the $24/month minimum allotment, $288/yr."
+us,scenario_031,snap,glm-5.3,llm_error,categorical_eligibility,False,"Passed the household on California's 200% FPL gross test but then applied the 100% FPL net income test. Broad-based categorical eligibility waives that test, so the household gets the $24/month minimum allotment instead of $0."
+us,scenario_031,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Treated countable income as very low and produced $144/month. Countable unearned income is $1,988/month and net income is about $1,611, so $298 minus 30% of net income ($484) is negative. Only the $24/month minimum allotment applies ($288/yr)."
+us,scenario_031,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Overstated income at about $31k (actual countable income is $23,853) and applied generic thresholds to find ineligibility. Gross income is under California's 200% FPL broad-based categorical eligibility limit, so the household is eligible for the $24/month minimum allotment."
+us,scenario_031,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,Returned zero because the computed allotment is negative after deductions. It omitted the $24/month minimum allotment owed to an eligible one-person household; this household is categorically eligible under California's 200% FPL broad-based categorical eligibility.
+us,scenario_031,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"Disqualified the household on the SNAP net income limit. Its gross income is under California's 200% FPL broad-based categorical eligibility limit, which waives the net test. The eligible one-person household receives the $24/month minimum allotment."
+us,scenario_031,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Applied the net income limit as disqualifying. California's broad-based categorical eligibility (gross under 200% FPL) waives the net test, and the eligible one-person household gets the $24/month minimum allotment ($288/yr)."
+us,scenario_031,snap,gpt-6-luna,llm_error,categorical_eligibility,False,Stopped at a zero computed allotment. It missed the $24/month minimum allotment for eligible one-person households; this household is eligible through California's 200% FPL broad-based categorical eligibility.
+us,scenario_031,snap,gpt-6-sol,llm_error,categorical_eligibility,False,Concluded no allotment remains after deductions. It ignored the $24/month minimum allotment that a categorically eligible one-person household receives regardless of how far 30% of net income exceeds the $298 maximum.
+us,scenario_031,snap,gpt-6.1-terra,llm_error,categorical_eligibility,False,"Treated income that is too high for a positive formula benefit as a $0 result. A one-person household that is categorically eligible under California's broad-based categorical eligibility receives the $24/month minimum allotment, $288/yr."
+us,scenario_031,snap,grok-4.3,llm_error,categorical_eligibility,False,"Applied a 130% FPL gross test and claimed assets exceed the limit. Elderly households are exempt from the 130% test, California's broad-based categorical eligibility uses 200% FPL, and that eligibility waives the asset test ($4,200 is also under the $4,500 elderly limit). The household gets the $24/month minimum."
+us,scenario_031,snap,grok-4.5,llm_error,categorical_eligibility,False,"Failed the household on the 100% FPL net income test. Categorical eligibility under California's 200% FPL broad-based categorical eligibility waives that test, so the one-person household receives the $24/month minimum allotment."
+us,scenario_031,snap,grok-4.6,llm_error,categorical_eligibility,False,"Correctly found that 30% of net income exceeds the maximum allotment but returned $0. It missed the $24/month minimum allotment for eligible one-person households, which yields $288/yr."
+us,scenario_031,snap,grok-4.7,llm_error,categorical_eligibility,False,"Disqualified the household on the net income limit after deductions. California's 200% FPL broad-based categorical eligibility waives the net test, and the eligible one-person household gets the $24/month minimum allotment."
+us,scenario_031,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"Barely reduced annual income for deductions (to about $23,628) and failed the household on a net income limit of about $16,000. Gross income is under California's 200% FPL broad-based categorical eligibility limit, which waives the net test. The household receives the $24/month minimum allotment ($288/yr)."
+us,scenario_031,snap,inkling,llm_error,categorical_eligibility,False,"Waived the gross test as an elderly household but applied the 100% FPL net test. California's broad-based categorical eligibility (gross $1,988 under the $2,608 limit) also waives the net test, giving the $24/month minimum allotment."
+us,scenario_031,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"Found the household ineligible on the net income limit. Categorical eligibility through California's 200% FPL broad-based categorical eligibility waives that limit, and the one-person household receives the $24/month minimum allotment."
+us,scenario_031,snap,kimi-k3,llm_error,categorical_eligibility,False,"Applied the one-person net income limit as disqualifying. California's broad-based categorical eligibility, met at $1,988/month gross, waives the net test, so the household gets the $24/month minimum allotment ($288/yr)."
+us,scenario_031,snap,minimax-m3,llm_error,asset_resource,False,"Used an outdated $3,500 elderly asset limit to deny eligibility. The federal elderly limit is $4,500, which $4,200 passes, and California's broad-based categorical eligibility waives the asset test anyway. The eligible household receives the $24/month minimum allotment."
+us,scenario_031,snap,muse-spark-1.3,llm_error,categorical_eligibility,False,"Treated the 100% FPL net limit as disqualifying. California's 200% FPL broad-based categorical eligibility waives the net test, and the eligible one-person household gets the $24/month minimum allotment ($288/yr)."
+us,scenario_031,snap,muse-wonder,llm_error,categorical_eligibility,False,"Denied eligibility because net income of about $1,779 exceeds 100% FPL. Under California's broad-based categorical eligibility that test does not apply, and the one-person household receives the $24/month minimum allotment."
+us,scenario_031,snap,ox-alpha,llm_error,categorical_eligibility,False,Correctly computed $298 minus 30% of net income as negative but returned $0. It omitted the $24/month minimum allotment for an eligible one-person household; this household is eligible through California's broad-based categorical eligibility.
+us,scenario_031,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"Applied a 130% FPL gross income test. That test does not apply to an elderly household, and California's broad-based categorical eligibility sets a 200% FPL ($2,608/month) limit that the $1,988 income passes. The household gets the $24/month minimum allotment."
+us,scenario_031,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"Used a miscomputed countable income of $19,765.50 (actual countable income is $23,853) and an unspecified income limit to deny eligibility. Gross income is under California's 200% FPL broad-based categorical eligibility limit, so the household gets the $24/month minimum allotment ($288/yr)."
us,scenario_031,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model explicitly applied no nonrefundable credits, omitting the California aged-or-blind exemption credit available to the 67-year-old filer. It also substituted an estimated $6,043 standard deduction for the applicable $5,706 deduction, but the decisive error is failing to reduce the resulting $75.37 tax by $312.93 of nonrefundable credits, which yields $0."
us,scenario_032,child1_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"It applied only an age test plus a ~200% FPL CHIP income screen to household income of $42,664 and never ran the Medicaid-first screen that CHIP requires: Minnesota's Medical Assistance covers children 6-18 (the engine's OLDER_CHILD category) well above this ~160% FPL household, and a Medicaid-eligible child is categorically excluded from CHIP. It also sized the household at four people when only three are listed, inflating the FPL denominator it compared against."
us,scenario_032,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"It named the correct Minnesota number — 275% FPL for children — but that band is Medical Assistance (Medicaid) for children ages 2-18, not CHIP, so computing ~160% FPL established Medicaid eligibility under the OLDER_CHILD category. Treating 'MN CHIP/Medicaid expansion thresholds' as one interchangeable ceiling let it skip the mutual-exclusivity step that makes a Medicaid-eligible child ineligible for CHIP."
@@ -3233,33 +3277,34 @@ us,scenario_032,federal_refundable_credits,muse-wonder,llm_error,credit_phaseout
us,scenario_032,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It used a joint-filer phase-out start of about $22,190, which leaves out the MFJ increase, and reduced the EITC to $3,348. The household's AGI is below the actual MFJ threshold, so the full $4,427 applies. Its $1,700 ACTC was correct."
us,scenario_032,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used inflated EITC maximums ($5,386–$5,640) with a phase-out start of about $22,000–$23,500. It then dropped its own $1,700 ACTC figure in favor of an unsupported $800. The correct values are a $4,427 EITC and a $1,700 ACTC."
us,scenario_032,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It phased the EITC down to $1,171 and cut the ACTC to $937. AGI of about $30,916 is below the MFJ one-child phase-out start, so the full $4,427 EITC applies, and 15% of earnings above $2,500 ($3,975) exceeds the $1,700 cap, so the ACTC is $1,700."
-us,scenario_032,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"Applied only the federal 130% FPL free-meal test and the SNAP/TANF categorical pathway. It never applied Minnesota's universal free school meals program, which puts the household's K-12 child in the FREE tier regardless of the $42,664 income."
-us,scenario_032,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"Concluded ineligible because income exceeds 130% FPL and no direct certification applies. It ignored that Minnesota provides universal free school meals, so the household's 6-year-old qualifies for FREE meals no matter the income."
-us,scenario_032,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"Treated the 130% FPL federal income limit as the only path to free meals. It missed Minnesota's universal free school meals law, which makes the K-12 child FREE-eligible at 156% FPL."
-us,scenario_032,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"Correctly computed $42,664 as above the federal 130% FPL threshold. However, it explicitly framed Minnesota eligibility through the federal NSLP income test only and missed that Minnesota has run universal free school meals since 2023, which gives the child FREE meals regardless of income."
-us,scenario_032,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"Compared countable income to 130% FPL and stopped there. It never applied Minnesota's universal free school meals program, which overrides the income test and puts the household in the FREE tier."
-us,scenario_032,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Used only the federal 130% FPL free-meal threshold, at roughly 157% FPL. It missed Minnesota's universal free school meals program, which makes the K-12 child FREE-eligible regardless of income."
-us,scenario_032,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"Compared $42,664 to a 130% FPL limit of $35,326 and concluded ineligible. It ignored Minnesota's statewide universal free school meals program, which grants FREE meals without any income test."
-us,scenario_032,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Applied only the federal 130% FPL income limit for a 3-person household. It missed that Minnesota provides universal free school meals, so the child qualifies for FREE meals despite the household being over the income limit."
-us,scenario_032,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"Explicitly assessed only federal free-meal eligibility against 130% FPL. It never considered Minnesota's universal free school meals law, which makes every K-12 student in the state FREE-eligible."
-us,scenario_032,free_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,"Considered only the federal 130% FPL test and categorical eligibility through SNAP/TANF. It missed the third pathway, Minnesota's universal free school meals program, which gives the household FREE-tier meals regardless of income."
-us,scenario_032,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"Correctly summed wages and Social Security to $42,664, then applied only the federal 130% FPL cutoff. It missed that Minnesota's universal free school meals program sets the tier to FREE for K-12 children regardless of income."
-us,scenario_032,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"Stopped at the federal 130% FPL cutoff, with income at about 153% FPL. It never applied Minnesota's universal free school meals program, which makes the K-12 child FREE-eligible."
-us,scenario_032,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed free meals require categorical SNAP/TANF eligibility and returned 0 because there was none. It ignored Minnesota's universal free school meals program, which gives FREE meals to the household's K-12 child with no income or categorical test."
-us,scenario_032,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"Applied only the federal free-meal income limit and categorical status. Although it noted the school-age child, it missed that Minnesota's universal free school meals program qualifies that child for FREE meals regardless of income."
-us,scenario_032,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"Relied on the federal free-meal income threshold alone. It missed Minnesota's universal free school meals program, under which PolicyEngine returns positive free-meal support for any K-12 child in the state."
-us,scenario_032,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"Applied the federal free-meal income threshold and direct certification only. It never applied Minnesota's universal free school meals rule, which assigns the FREE tier regardless of income."
-us,scenario_032,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"Concluded no free-meal support because income exceeds the free-meal limit. It ignored that Minnesota offers universal free school meals, which makes the household's K-12 child FREE-eligible at any income."
-us,scenario_032,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"Correctly placed income at about 156% FPL but treated the federal 130% limit as decisive. It missed Minnesota's universal free school meals program, which overrides the income test."
-us,scenario_032,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"Applied the 130% FPL limit and SNAP/TANF categorical eligibility, then asserted PolicyEngine returns no free-meal support. In fact, PolicyEngine flags Minnesota as a universal free school meals state and sets the tier to FREE regardless of income."
-us,scenario_032,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"Checked only the federal 130% FPL limit and the SNAP/TANF categorical pathway. It never applied Minnesota's universal free school meals program, which makes the child FREE-eligible at 156% FPL."
-us,scenario_032,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,Compared income to 130% of a 2026 FPL for a 3-person household and concluded ineligible. It missed that Minnesota's universal free school meals program grants FREE meals with no income test.
-us,scenario_032,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,"Used only the federal 130% FPL free-meal limit, which it put at $33,566. It ignored Minnesota's universal free school meals law, which places the household's K-12 child in the FREE tier regardless of income."
-us,scenario_032,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"No answer or explanation was returned for free_school_meals_eligible, so the output was missing rather than wrong on substance. The correct value is 1, because Minnesota's universal free school meals program covers the K-12 child."
-us,scenario_032,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"Applied only what it called PolicyEngine's federal school-meal income test, with income at about 156% of poverty. It missed that PolicyEngine also checks state_has_universal_free_school_meals, which is True for Minnesota and sets the tier to FREE."
-us,scenario_032,free_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,"Compared gross income to the 130% FPL free-meal limit only. It never applied Minnesota's universal free school meals program, which qualifies the K-12 child for FREE meals regardless of income."
-us,scenario_032,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"Treated the 130% of poverty limit as the only route to free meals. It missed Minnesota's universal free school meals program, which gives the household positive free-meal support despite income above the limit."
-us,scenario_032,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,Correctly found income above the 130% cutoff and no SNAP/TANF categorical pathway. It overlooked the state pathway: Minnesota's universal free school meals program sets the FREE tier for K-12 students regardless of income.
+us,scenario_032,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model correctly found income above 130% FPL with no SNAP/TANF categorical pathway, but then stopped. It missed that Minnesota's universal free school meals program gives free meals to every K-12 student regardless of income, so the K-12 child qualifies."
+us,scenario_032,free_school_meals_eligible,claude-fable-5.2,llm_error,state_local_rule,False,"The model treated the federal 130% FPL test and SNAP/TANF categorical eligibility as the only ways to get free meals. It missed Minnesota's universal free school meals program, which makes the K-12 child eligible for free meals at 156% FPL."
+us,scenario_032,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model overstated the income ratio (about 170% FPL; the engine computes 1.56). Its main error was using only the federal 130% FPL and direct-certification tests while ignoring Minnesota's universal free school meals program, which makes the household eligible for free meals regardless of income."
+us,scenario_032,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"The model applied the federal 130% FPL free-meal limit as decisive. It never considered that Minnesota provides universal free school meals to all K-12 students, which makes the 6-year-old eligible for free meals."
+us,scenario_032,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,The model explicitly framed Minnesota eligibility through the federal NSLP 130% FPL test and missed that Minnesota has had universal free school meals since 2023. Under that program the K-12 child receives free meals regardless of the household's 156% FPL income.
+us,scenario_032,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model compared $42,664 with the federal 130% FPL limit and concluded the household was ineligible. It missed Minnesota's universal free school meals program, which gives free meals to all K-12 students regardless of income."
+us,scenario_032,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model applied only the federal 130% FPL free-meal threshold to the 157% FPL income. It ignored Minnesota's universal free school meals program, which overrides the income test and makes the K-12 child eligible for free meals."
+us,scenario_032,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,The model concluded the household was ineligible because income exceeded 130% FPL. It missed that Minnesota's universal free school meals program gives every K-12 student free meals regardless of household income.
+us,scenario_032,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model used only the federal 130% FPL income test. It overlooked Minnesota's universal free school meals program, which sets the school meal tier to FREE for the K-12 child even at 156% FPL."
+us,scenario_032,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"The model's reasoning was limited to federal free-meal eligibility at 130% FPL. It missed that Minnesota provides universal free school meals, which makes the K-12 child eligible for free meals whatever the household's income."
+us,scenario_032,free_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,"The model checked only the federal 130% FPL test and categorical eligibility, and correctly found the household failed both. It missed the third pathway: Minnesota's universal free school meals program, which makes the household eligible."
+us,scenario_032,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model counted the $42,664 of income correctly but applied only the federal 130% FPL free-meal limit. It ignored Minnesota's universal free school meals program, which gives the K-12 child free meals regardless of income."
+us,scenario_032,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model treated the federal 130% cutoff as decisive at about 153% FPL. It missed Minnesota's universal free school meals program, under which the K-12 child qualifies for free meals without an income test."
+us,scenario_032,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"The model answered no because there was no SNAP/TANF categorical eligibility and it never ran a state-level check. It missed that Minnesota's universal free school meals program gives free meals to all K-12 students, including this 6-year-old."
+us,scenario_032,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model applied the federal free-meal income limit and found no categorical status. It overlooked Minnesota's universal free school meals program, which makes the household eligible for free meals regardless of income."
+us,scenario_032,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model's own reasoning shows it relied only on the federal free-meal threshold. It missed that Minnesota provides universal free school meals, so the K-12 child receives free meals even though income exceeds 130% FPL."
+us,scenario_032,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model applied the federal income threshold and the direct-certification pathway, and the household failed both. It never applied Minnesota's universal free school meals program, which sets the household's tier to FREE."
+us,scenario_032,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"The model concluded that income above the free-meal threshold ruled out eligibility. It ignored Minnesota's universal free school meals program, which gives the K-12 child free meals with no income test."
+us,scenario_032,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model calculated about 156% FPL, which matches the engine, but treated the federal 130% limit as final. It missed Minnesota's universal free school meals program, which makes the K-12 child eligible for free meals anyway."
+us,scenario_032,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,The model correctly found 156% FPL and no SNAP/TANF categorical eligibility. It wrongly concluded that PolicyEngine returns no free-meal support: PolicyEngine sets the tier to FREE because Minnesota has universal free school meals.
+us,scenario_032,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"The model checked only the federal 130% FPL limit and SNAP/TANF categorical eligibility. It missed that Minnesota's universal free school meals program makes every K-12 student, including this child, eligible for free meals regardless of income."
+us,scenario_032,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model's FPL ratio of about 156.7% matched the engine, but it treated exceeding 130% FPL as disqualifying. It overlooked Minnesota's universal free school meals program, which makes the household eligible for free meals."
+us,scenario_032,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,"The model used only the federal 130% FPL free-meal limit. It missed that Minnesota provides universal free school meals to all K-12 students, so the 6-year-old qualifies regardless of the household's $42,664 income."
+us,scenario_032,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for free_school_meals_eligible, so the output is missing. The correct answer is yes, because Minnesota's universal free school meals program covers the K-12 child."
+us,scenario_032,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model claimed to follow PolicyEngine's rules but applied only the federal income test at 156% FPL. PolicyEngine also checks state_has_universal_free_school_meals, which is True for Minnesota, and that sets the tier to FREE."
+us,scenario_032,free_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,"The model concluded the household was ineligible because gross income exceeded 130% FPL. It missed Minnesota's universal free school meals program, which gives the K-12 child free meals regardless of income."
+us,scenario_032,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"The model applied the federal 130% of poverty limit as the only free-meal test. It ignored Minnesota's universal free school meals program, which makes the household eligible for positive free-meal support."
+us,scenario_032,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model correctly counted the Social Security income, ruled out the 130% cutoff and found no SNAP/TANF pathway. It missed Minnesota's universal free school meals program, which provides free meals to every K-12 student regardless of income."
us,scenario_032,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_032,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model claimed Social Security benefits are excluded from MAGI and counted only the ~$29,000 in wages. Medicaid MAGI adds back the non-taxable portion of Social Security, so the full $13,664 counts. That makes MAGI $42,664, about 156% FPL, which is above the 138% limit."
us,scenario_032,head_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly counted $42,664 including Social Security and saw that this is above 138% FPL. It then made the head eligible through a higher Minnesota threshold for parents and caretakers, which does not exist. Minnesota Medical Assistance covers parents at the same 133%+5% (138%) FPL limit, and MinnesotaCare is a Basic Health Program, not Medicaid."
@@ -3279,79 +3324,81 @@ us,scenario_032,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_dedu
us,scenario_032,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model first counted $42,664 and then dropped the Social Security benefits on the theory that a child's benefits are excluded. The benefits are listed as the head's income, so they count in full in the head's MAGI. That puts income at 156% FPL, above the 138% limit."
us,scenario_032,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_032,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a household-level “WIC-eligible context” pathway and transferred eligibility to the head based on living with a child. The head is age 50, is not pregnant, postpartum, or breastfeeding, and therefore fails WIC's person-specific categorical requirement; moreover, the only child is age 6 and is not categorically eligible either."
-us,scenario_032,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"The model computed FICA correctly at $2,218.50. It said Minnesota has no mandatory employee state payroll tax in 2026, so it left out the 0.44% Minnesota Paid Leave employee contribution ($127.60) that PolicyEngine adds by assuming the employer withholds the maximum employee share."
-us,scenario_032,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"The model applied only the 7.65% FICA rate. It said Minnesota has no modeled mandatory employee payroll tax, so it dropped the 0.44% Minnesota Paid Leave employee contribution on $29,000 ($127.60) that PolicyEngine includes."
-us,scenario_032,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"The model correctly worked out FICA of $2,218.50, then for no stated reason changed it to $2,175.90 with a vague 'state-specific considerations' adjustment. It still missed the $127.60 Minnesota Paid Leave employee contribution (0.44% × $29,000) that brings the total to $2,346.10."
-us,scenario_032,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"The model said Minnesota has no mandatory employee state payroll tax, so it left out the 0.44% Minnesota Paid Leave employee contribution ($127.60). It also rounded the FICA total of $2,218.50 to $2,219."
-us,scenario_032,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"The model applied only the 7.65% FICA rate and said Minnesota has no mandatory employee payroll tax. That left out the 0.44% Minnesota Paid Leave employee contribution on $29,000 ($127.60), which PolicyEngine counts as withheld from the employee."
-us,scenario_032,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"The model computed the $1,798 Social Security and $420.50 Medicare correctly. It treated Minnesota as having no employee payroll tax and so left out the $127.60 Minnesota Paid Leave employee contribution (0.44% of wages), which started in 2026."
-us,scenario_032,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"The model stopped at FICA ($2,218.50) because it said there is no mandatory Minnesota employee payroll tax. That omitted the 0.44% Minnesota Paid Leave employee contribution ($127.60) included in the reference."
-us,scenario_032,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"The model applied only the 7.65% FICA rate to $29,000. It never added the 0.44% Minnesota Paid Leave employee contribution ($127.60) that PolicyEngine counts as a state payroll tax withheld from the employee."
-us,scenario_032,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"The model said Minnesota imposes no employee payroll tax beyond the federal ones. That missed the Minnesota Paid Leave premium starting in 2026, whose 0.44% employee share on $29,000 adds $127.60."
-us,scenario_032,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"The model said Minnesota has no paid family leave employee contribution in 2026, which is wrong. The Minnesota Paid Leave employee share of 0.44% applies to the $29,000 in wages and adds $127.60 to the $2,218.50 in FICA."
-us,scenario_032,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"The model assumed no Minnesota employee payroll tax and stopped at FICA of $2,218.50. That omitted the 0.44% Minnesota Paid Leave employee contribution ($127.60), which PolicyEngine includes by assuming the employer withholds the maximum employee share."
-us,scenario_032,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,The model said no employee state payroll tax applies and charged only the 7.65% FICA. That left out the 0.44% Minnesota Paid Leave employee contribution of $127.60.
-us,scenario_032,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"The model correctly included the Minnesota paid leave employee share but used a 0.35% rate, giving $101.50. The 2026 rate is 0.44%, half of the 0.88% total premium, which gives $127.60 and a total of $2,346.10."
-us,scenario_032,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"The model's explanation describes FICA only ($2,218.50), but it submitted $2,320. That figure matches adding a Minnesota paid leave employee share at a wrong 0.35% rate ($101.50) instead of the 0.44% rate that gives $127.60."
-us,scenario_032,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"The model applied only the 7.65% FICA rate to $29,000. It did not add the 0.44% Minnesota Paid Leave employee contribution ($127.60) that PolicyEngine counts toward payroll tax."
-us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"The model computed only the employee Social Security and Medicare shares, rounded to $2,219. It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60)."
-us,scenario_032,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"The model applied only 7.65% FICA to $29,000. It never considered the Minnesota Paid Leave premium, whose 0.44% employee share adds $127.60."
-us,scenario_032,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"The model counted only FICA at 7.65% of $29,000. It omitted the 0.44% Minnesota Paid Leave employee contribution ($127.60) that PolicyEngine includes as a state payroll tax."
-us,scenario_032,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"The model counted only Social Security and Medicare on $29,000. It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60)."
-us,scenario_032,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"The model applied only the 7.65% FICA rate. It never added the 0.44% Minnesota Paid Leave employee contribution on $29,000 ($127.60)."
-us,scenario_032,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"The model computed FICA only ($2,218.50). It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60) that PolicyEngine treats as withheld from wages in 2026."
-us,scenario_032,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"The model summed $1,798 in Social Security and $420.50 in Medicare but never added the 0.44% Minnesota Paid Leave employee contribution ($127.60)."
-us,scenario_032,payroll_tax,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"The model counted only FICA at 7.65%. It did not include the 0.44% Minnesota Paid Leave employee contribution on $29,000 ($127.60) that is part of the reference total."
-us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so nothing could be scored. The correct total is $2,346.10: $1,798 in Social Security, $420.50 in Medicare and $127.60 for Minnesota Paid Leave."
-us,scenario_032,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,The model applied 7.65% FICA and checked the Additional Medicare Tax but never considered state payroll levies. That omitted the 0.44% Minnesota Paid Leave employee contribution ($127.60).
-us,scenario_032,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"The model's explanation describes the employee 6.2% + 1.45% rates, but it submitted $4,428.60. That is about twice the employee FICA, consistent with wrongly adding the employer share. It also omitted the $127.60 Minnesota Paid Leave employee contribution."
-us,scenario_032,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"The model correctly computed 7.65% × $29,000 = $2,218.50, then dropped it for a made-up 5.24% rate based on a nonexistent wage-base interaction, giving $1,520. It also left out the $127.60 Minnesota Paid Leave employee contribution."
-us,scenario_032,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"The model said there is no state payroll tax and stopped at $2,218.50 in FICA. That omitted the 0.44% Minnesota Paid Leave employee contribution ($127.60)."
-us,scenario_032,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"The model applied only 7.65% FICA, rounded to $2,219. It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60)."
-us,scenario_032,payroll_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"The model said no other employee-side payroll tax applies and stopped at 7.65% FICA. That omitted the 0.44% Minnesota Paid Leave employee contribution on $29,000 ($127.60)."
-us,scenario_032,payroll_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"The model counted only the 7.65% FICA on $29,000. It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60) included in the reference."
-us,scenario_032,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,The model applied only the employee Social Security and Medicare rates. It never added the 0.44% Minnesota Paid Leave employee contribution ($127.60).
-us,scenario_032,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"The model said no mandatory Minnesota employee payroll tax applies. That excluded the 0.44% Minnesota Paid Leave employee contribution ($127.60), which PolicyEngine treats as withheld in 2026."
-us,scenario_032,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"The model said Minnesota has no employee-side payroll tax. That missed the Minnesota Paid Leave premium starting in 2026, whose 0.44% employee share on $29,000 is $127.60."
-us,scenario_032,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"The model recognized the Minnesota Paid Leave program but excluded it because the employer can choose not to withhold the employee share. PolicyEngine assumes the employer withholds the maximum 0.44% employee share, which adds $127.60."
-us,scenario_032,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"The model computed only employee FICA on $29,000. It never added the 0.44% Minnesota Paid Leave employee contribution ($127.60)."
-us,scenario_032,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"The model said no state employee payroll taxes apply. That omitted the 0.44% Minnesota Paid Leave employee contribution on $29,000 ($127.60)."
-us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so nothing could be scored. The correct total is $2,346.10: $1,798 in Social Security, $420.50 in Medicare and $127.60 for Minnesota Paid Leave."
-us,scenario_032,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"The model computed FICA only and rounded it to $2,219. It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60)."
-us,scenario_032,payroll_tax,muse-spark-1.3,prompt_ambiguity,state_local_rule,False,"The model summed only Social Security ($1,798) and Medicare ($420.50). It omitted the 0.44% Minnesota Paid Leave employee contribution ($127.60)."
-us,scenario_032,payroll_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"The model applied only 7.65% FICA. It did not add the 0.44% Minnesota Paid Leave employee contribution on $29,000 ($127.60) that PolicyEngine counts as a state payroll tax."
-us,scenario_032,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"The model said Minnesota levies no mandatory employee payroll tax. That excluded the 0.44% Minnesota Paid Leave employee contribution ($127.60), which starts in 2026."
-us,scenario_032,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"The model said Minnesota has no state employee payroll tax like California's or New York's. That missed the Minnesota Paid Leave premium, whose 0.44% employee share adds $127.60. It also rounded FICA to $2,219."
-us,scenario_032,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"The model counted only 7.65% FICA (its $419.50 Medicare figure is a typo, but the $2,218.50 total is right). It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60)."
-us,scenario_032,reduced_price_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model placed income of $42,664 between 130% and 185% FPL for 3 people and concluded the household gets reduced-price meals. It ignored Minnesota's universal free school meals program, which gives every student the FREE tier, so reduced-price eligibility is 0."
-us,scenario_032,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model applied only the federal 185% FPL reduced-price test, and it also miscounted the household as 4 people with 2 children when there are 3. It missed that Minnesota's universal free school meals program puts the household in the FREE tier, which replaces reduced-price eligibility."
-us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model correctly put $42,664 in the federal 130-185% FPL band but stopped at the federal tiers. Minnesota's universal free school meals program assigns the FREE tier to all students, so the household is not reduced-price eligible."
-us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model used only the federal 130%/185% poverty bands to place the 6-year-old in reduced-price meals. Minnesota's statewide universal free meals override those bands and give the household the FREE tier, so reduced-price eligibility is 0."
-us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model assigned reduced-price meals from an income of about 170% FPL alone. It did not apply Minnesota's universal free school meals program, which gives the household the FREE tier instead."
-us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"The model treated income of about 160% FPL as automatically reduced-price under federal NSLP bands. It missed that Minnesota provides free meals to all students, which places the household in the FREE tier and zeroes out reduced-price eligibility."
-us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model went back and forth over exact FPL figures for the 130-185% band but never considered state policy. Minnesota's universal free school meals program gives the household the FREE tier whatever its income, so reduced-price eligibility is 0."
-us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model placed income of about $42,700 between 130% and 185% FPL and declared reduced-price eligibility. It ignored Minnesota's universal free school meals program, which assigns the FREE tier and replaces reduced-price status."
-us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model used only the federal 130-185% FPL reduced-price band at about 157% FPL. It missed that Minnesota's universal free school meals program gives all students free meals, which makes reduced-price eligibility 0."
-us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model checked only that $42,664 is under 185% FPL and concluded reduced-price eligibility. It never applied Minnesota's universal free meals program, which assigns the household the FREE tier instead."
-us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model relied on the federal 130%/185% FPL income bands alone. Minnesota's universal free school meals program places the household in the FREE tier, so reduced-price eligibility is 0."
-us,scenario_032,reduced_price_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"The model placed $42,664 in the federal reduced-price band for 3 people without checking state policy. Minnesota's universal free school meals give every student the FREE tier, which replaces reduced-price eligibility."
-us,scenario_032,reduced_price_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,"The model applied only the federal 130-185% FPL test. It missed Minnesota's universal free school meals program, under which PolicyEngine assigns the FREE tier and returns no reduced-price support."
-us,scenario_032,reduced_price_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model concluded reduced-price eligibility because income falls between 130% and 185% of the 2026 FPL. It ignored that Minnesota provides universal free school meals, which puts the household in the FREE tier."
-us,scenario_032,reduced_price_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model used the federal reduced-price band at about 153% FPL and stopped there. Minnesota's universal free school meals program assigns the FREE tier to all students, so reduced-price eligibility is 0."
-us,scenario_032,reduced_price_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model compared income only to the federal free and reduced-price limits for 3 people. It missed Minnesota's universal free school meals program, which overrides those limits and assigns the FREE tier."
-us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model applied only the federal reduced-price income threshold for 3 people. Minnesota's universal free school meals program gives the household the FREE tier, so PolicyEngine returns no reduced-price support."
-us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model placed household income in the modeled reduced-price range using federal bands alone. It did not account for Minnesota's universal free school meals program, which puts the household in the FREE tier."
-us,scenario_032,reduced_price_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"The model assigned reduced-price meals from the federal income range for 3 people. Minnesota's universal free school meals program gives all students free meals, which zeroes out reduced-price eligibility."
-us,scenario_032,reduced_price_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model correctly computed income at about 156% FPL but applied only the federal 185% reduced-price limit. It missed Minnesota's universal free school meals program, which assigns the FREE tier."
-us,scenario_032,reduced_price_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,The model placed income at about 156% FPL in the federal 130-185% band and returned reduced-price eligibility. PolicyEngine applies Minnesota's universal free school meals and assigns the FREE tier instead.
-us,scenario_032,reduced_price_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"The model used only the federal 130%/185% FPL bands for a school-age child. It ignored Minnesota's universal free school meals program, which gives the household the FREE tier and makes reduced-price eligibility 0."
-us,scenario_032,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,The model computed about 156.7% FPL and applied the federal 185% reduced-price threshold. It missed that Minnesota's universal free school meals program places the household in the FREE tier.
-us,scenario_032,reduced_price_school_meals_eligible,inkling,llm_error,state_local_rule,False,"The model placed $42,664 between federal 130% and 185% FPL dollar limits ($33,566-$47,767) and concluded reduced-price. It never applied Minnesota's universal free school meals program, which assigns the FREE tier to all students."
-us,scenario_032,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no parseable value or explanation for reduced_price_school_meals_eligible, so no substantive answer was recorded. The correct answer is 0 because Minnesota's universal free school meals program puts the household in the FREE tier."
-us,scenario_032,reduced_price_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model treated the federal 130% free-meal limit as decisive and placed income at about 156% FPL in the reduced-price band. Minnesota's universal free school meals program makes the household FREE regardless of income, so reduced-price eligibility is 0."
-us,scenario_032,reduced_price_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,"The model applied only the federal 130-185% FPL reduced-price band. It missed Minnesota's universal free school meals program, which assigns the FREE tier and replaces reduced-price eligibility."
-us,scenario_032,reduced_price_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"The model treated the household as above the free-meal limit using federal income tests alone. Minnesota's universal free school meals program removes that limit and puts the household in the FREE tier, so reduced-price support is 0."
-us,scenario_032,reduced_price_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model placed income at about 160% FPL under the federal 185% reduced-price cutoff. It ignored Minnesota's universal free school meals program, which assigns the FREE tier instead of reduced-price."
+us,scenario_032,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"Computed federal FICA correctly ($1,798 + $420.50) but said Minnesota has no mandatory employee state payroll tax in 2026. It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60 on $29,000), which PolicyEngine counts by assuming maximum employee withholding."
+us,scenario_032,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"Applied 7.65% FICA to $29,000 and assumed no Minnesota employee payroll tax. It left out the 0.44% Minnesota Paid Leave employee premium ($127.60), which PolicyEngine includes by assuming the employer withholds the maximum employee share."
+us,scenario_032,payroll_tax,claude-fable-5.2,prompt_ambiguity,state_local_rule,False,"Included only federal Social Security and Medicare tax. It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60), which became effective in 2026 and which PolicyEngine treats as an employee state payroll tax."
+us,scenario_032,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"Correctly derived $2,218.50 in FICA, then cut it to $2,175.90 for unspecified 'state-specific considerations' with no basis. The adjustment points the wrong way: Minnesota adds the 0.44% Paid Leave employee contribution ($127.60), for a total of $2,346.10."
+us,scenario_032,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Rounded 7.65% FICA to $2,219 and said Minnesota has no employee state payroll tax. That leaves out the $127.60 Minnesota Paid Leave employee contribution (0.44% of $29,000) that PolicyEngine includes."
+us,scenario_032,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"Said Minnesota has no mandatory employee state payroll tax. That leaves out the 0.44% Minnesota Paid Leave employee premium ($127.60), which PolicyEngine adds to the $2,218.50 in FICA by assuming maximum employee withholding."
+us,scenario_032,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"Stopped at federal FICA, saying Minnesota has no mandatory employee payroll tax. It left out the 0.44% Minnesota Paid Leave employee contribution on $29,000 ($127.60) that started in 2026."
+us,scenario_032,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Treated Minnesota as having no mandatory employee payroll tax and left out the $127.60 Paid Leave employee contribution (0.44% × $29,000). PolicyEngine counts this contribution as an employee state payroll tax."
+us,scenario_032,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"Applied only 7.65% FICA to $29,000. It never considered the Minnesota Paid Leave employee share (0.44%, $127.60) that PolicyEngine adds as a state payroll tax."
+us,scenario_032,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"Said Minnesota imposes no employee-side payroll tax beyond federal FICA. It missed the Minnesota Paid Leave program's 0.44% employee premium ($127.60 on $29,000), which became effective in 2026 and which PolicyEngine includes."
+us,scenario_032,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"Said Minnesota has no paid family leave employee contribution in 2026. Minnesota Paid Leave premiums began January 1, 2026, and PolicyEngine adds the 0.44% employee share ($127.60) to FICA."
+us,scenario_032,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"Assumed no Minnesota employee payroll tax is modeled. PolicyEngine models the 0.44% Minnesota Paid Leave employee contribution ($127.60 on $29,000) and includes it in payroll_tax."
+us,scenario_032,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"Said no employee state payroll tax applies and left out the 0.44% Minnesota Paid Leave employee contribution ($127.60). PolicyEngine adds that contribution to the $2,218.50 in FICA."
+us,scenario_032,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Correctly included the Minnesota Paid Leave employee share but used 0.35%, half of the original 0.7% statutory premium, giving $101.50. The 2026 premium is 0.88%, so the employee share is 0.44%, which gives $127.60 and a total of $2,346.10."
+us,scenario_032,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"Submitted $2,320, which equals FICA plus a 0.35% Minnesota Paid Leave employee share ($101.50), while its explanation says no state payroll tax applies. The 2026 employee rate is 0.44% ($127.60), which gives $2,346.10."
+us,scenario_032,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"Applied only 7.65% FICA to $29,000 and left out the 0.44% Minnesota Paid Leave employee contribution ($127.60). PolicyEngine includes that contribution as an employee state payroll tax."
+us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"Counted only the Social Security and Medicare employee shares, rounded to $2,219. It left out the $127.60 Minnesota Paid Leave employee premium (0.44% of wages) that PolicyEngine adds."
+us,scenario_032,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,Applied only the 7.65% federal employee rate and left out the 0.44% Minnesota Paid Leave employee contribution ($127.60). PolicyEngine includes it by assuming maximum employee withholding.
+us,scenario_032,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"Computed only FICA at 7.65% and never added the Minnesota Paid Leave employee share (0.44% × $29,000 = $127.60). PolicyEngine counts that share as a state employee payroll tax."
+us,scenario_032,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,Reported only Social Security and Medicare tax. It left out the $127.60 Minnesota Paid Leave employee contribution at the 2026 rate of 0.44% of wages.
+us,scenario_032,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"Applied 7.65% to wages and stopped there. It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60), which PolicyEngine includes in payroll_tax."
+us,scenario_032,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Counted only employee FICA and left out the $127.60 Minnesota Paid Leave employee premium (0.44% of $29,000) that PolicyEngine adds as a state payroll tax."
+us,scenario_032,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"Correctly summed $1,798 + $420.50 but left out the 0.44% Minnesota Paid Leave employee contribution ($127.60), which started in 2026."
+us,scenario_032,payroll_tax,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"Treated payroll tax as FICA only and never applied the 0.44% Minnesota Paid Leave employee rate to the $29,000 in wages ($127.60). PolicyEngine includes that rate in employee state payroll tax."
+us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"Returned no usable payroll_tax value or explanation. The correct derivation is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave employee contribution = $2,346.10."
+us,scenario_032,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,Computed only 7.65% FICA and left out the 0.44% Minnesota Paid Leave employee contribution ($127.60). PolicyEngine adds that contribution by assuming maximum employee withholding.
+us,scenario_032,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Its explanation describes 6.2% + 1.45% employee FICA on $29,000 ($2,218.50), but it submitted $4,428.60. That is roughly double, consistent with folding in employer-matched FICA, which the question excludes. It also left out the $127.60 Minnesota Paid Leave employee contribution."
+us,scenario_032,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"Correctly computed 7.65% × $29,000 = $2,218.50, then dropped it for an invented 5.24% rate, citing a nonexistent Social Security wage-base interaction ($29,000 is far below the cap). It also left out the 0.44% Minnesota Paid Leave employee contribution ($127.60)."
+us,scenario_032,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"Said no state payroll tax applies. That leaves out the 0.44% Minnesota Paid Leave employee contribution ($127.60), which PolicyEngine counts on top of the $2,218.50 in FICA."
+us,scenario_032,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Applied only 7.65% FICA, rounded to $2,219, and left out the $127.60 Minnesota Paid Leave employee premium (0.44% of wages) that PolicyEngine includes."
+us,scenario_032,payroll_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"Concluded that no other employee-side payroll tax is indicated. Minnesota Paid Leave imposes a 0.44% employee premium in 2026 ($127.60 on $29,000), which PolicyEngine adds to FICA."
+us,scenario_032,payroll_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,Counted only Social Security and Medicare at 7.65% and left out the 0.44% Minnesota Paid Leave employee contribution ($127.60) that PolicyEngine includes.
+us,scenario_032,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"Applied only the federal 6.2% and 1.45% employee rates. It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60) on the head's $29,000 in wages."
+us,scenario_032,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"Said no mandatory Minnesota employee payroll tax applies. It left out the 0.44% Minnesota Paid Leave employee premium ($127.60), which PolicyEngine includes by assuming the employer withholds the maximum employee share."
+us,scenario_032,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"Said Minnesota has no employee-side payroll tax. It left out the 0.44% Paid Leave employee contribution ($127.60), which started in 2026 and which PolicyEngine includes."
+us,scenario_032,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"Correctly noted that Minnesota Paid Leave withholding is optional for the employer, so it excluded the contribution as not mandatory. PolicyEngine instead assumes maximum employee withholding and adds the 0.44% share ($127.60); this is exactly the ambiguity in the question's 'mandatory' wording."
+us,scenario_032,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,Summed only Social Security and Medicare employee tax. It never added the 0.44% Minnesota Paid Leave employee contribution ($127.60) that PolicyEngine counts as employee state payroll tax.
+us,scenario_032,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"Said no state employee payroll taxes apply. That leaves out the $127.60 Minnesota Paid Leave employee contribution (0.44% × $29,000) included in PolicyEngine's payroll_tax."
+us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no usable payroll_tax value or explanation. The correct derivation is $1,798 Social Security + $420.50 Medicare + $127.60 Minnesota Paid Leave employee contribution = $2,346.10."
+us,scenario_032,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"Rounded FICA to $2,219 and never considered Minnesota payroll taxes. It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60) that PolicyEngine adds."
+us,scenario_032,payroll_tax,muse-spark-1.3,prompt_ambiguity,state_local_rule,False,"Counted only the 6.2% and 1.45% federal employee taxes and left out the 0.44% Minnesota Paid Leave employee premium ($127.60), which PolicyEngine includes."
+us,scenario_032,payroll_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"Applied only 7.65% FICA to $29,000. It left out the $127.60 Minnesota Paid Leave employee contribution that PolicyEngine counts under the maximum-employee-withholding assumption."
+us,scenario_032,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"Said Minnesota levies no mandatory employee payroll tax. That leaves out the 0.44% Minnesota Paid Leave employee premium ($127.60 on $29,000), which started in 2026 and which PolicyEngine includes."
+us,scenario_032,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"Reasoned that Minnesota has no state disability tax like California or New York and stopped at FICA. It missed Minnesota's own Paid Leave program, whose 0.44% employee premium ($127.60) PolicyEngine includes."
+us,scenario_032,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"Reported 7.65% FICA (miswriting Medicare as $419.50 while still totaling $2,218.50). It left out the 0.44% Minnesota Paid Leave employee contribution ($127.60) that PolicyEngine adds."
+us,scenario_032,reduced_price_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model placed the household in the reduced-price tier because income was between 130% and 185% FPL. It never applied Minnesota's universal free school meals program, which puts every MN student in the FREE tier and rules out the reduced-price tier."
+us,scenario_032,reduced_price_school_meals_eligible,claude-fable-5.2,llm_error,state_local_rule,False,"The model used only the federal 130–185% band and concluded the household gets reduced-price meals. It missed that Minnesota provides universal free school meals, so the engine assigns the FREE tier and reduced-price support is zero."
+us,scenario_032,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model counted a 4-person household that does not exist (it added a second child) and tested income only against the federal 185% FPL limit. It ignored Minnesota's universal free school meals program, which gives the child FREE-tier meals and makes reduced-price eligibility 0."
+us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model concluded that income above 130% and below 185% FPL means reduced-price eligibility. It never accounted for Minnesota's universal free school meals law, which puts the household in the FREE tier and supersedes the reduced-price tier."
+us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model placed the 6-year-old in the federal reduced-price band by income alone. It did not apply Minnesota's universal free school meals program, under which all MN students receive free meals, so reduced-price support is 0."
+us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model applied the federal 130–185% FPL reduced-price band, and it estimated about 170% FPL when the engine ratio is 1.56. It missed that Minnesota's universal free school meals program assigns the FREE tier to every student, which excludes the reduced-price tier."
+us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"The model used the federal income bands and found the household between 130% and 185% FPL. It ignored that Minnesota provides universal free school meals, so the engine assigns the FREE tier and reduced-price eligibility is 0."
+us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model repeatedly checked income against the federal 130% and 185% FPL thresholds. It never considered Minnesota's universal free school meals program, which overrides income tiers and puts the child in the FREE tier, so reduced-price support is 0."
+us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model found income between 130% and 185% FPL for 3 and called the household reduced-price eligible. It missed Minnesota's universal free school meals program, which gives the household the FREE tier instead."
+us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model used only the federal reduced-price band of about 157% FPL. It did not apply Minnesota's universal free school meals program, under which the engine assigns the FREE tier and returns no reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model compared income only with the 185% FPL reduced-price cap. It ignored that Minnesota's universal free school meals program puts all students in the FREE tier, which excludes reduced-price eligibility."
+us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model applied the federal 130–185% FPL test and concluded reduced-price eligibility. It missed that Minnesota provides universal free school meals, so the engine's school_meal_tier is FREE and reduced-price is 0."
+us,scenario_032,reduced_price_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"The model treated the federal 130–185% FPL band as decisive. It never applied Minnesota's universal free school meals program, which assigns the FREE tier regardless of income and so rules out reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,"The model used only federal NSLP income bands to place the household in reduced-price. It overlooked Minnesota's universal free school meals law, which makes the household FREE-tier and reduced-price eligibility 0."
+us,scenario_032,reduced_price_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,The model found income between 130% and 185% of the guideline and called it reduced-price. It ignored that Minnesota's universal free school meals program assigns every student the FREE tier.
+us,scenario_032,reduced_price_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model applied the federal 130–185% band (it estimated about 153% FPL). It missed Minnesota's universal free school meals program, which puts the household in the FREE tier and zeroes out reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model applied the federal free and reduced-price income limits only. It did not apply Minnesota's universal free school meals program, which supersedes those limits by placing all MN students in the FREE tier."
+us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model relied on the federal reduced-price income threshold for three people. It ignored Minnesota's universal free school meals program, under which the engine assigns FREE meals and returns no reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,The model placed the household in the reduced-price income range using federal limits. It missed that Minnesota's universal free school meals program overrides income tiers with the FREE tier.
+us,scenario_032,reduced_price_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"The model assigned the reduced-price tier from income alone. It never applied Minnesota's universal free school meals program, which gives the household FREE meals and therefore 0 reduced-price eligibility."
+us,scenario_032,reduced_price_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model correctly computed about 156% FPL but treated the federal 185% limit as decisive. It missed that Minnesota's universal free school meals program assigns the FREE tier, which excludes reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"The model placed income of about 156% FPL in the federal reduced-price band. It did not apply Minnesota's universal free school meals program, under which PolicyEngine classifies the household as FREE tier and returns zero reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"The model used only the federal 130–185% FPL test. It ignored Minnesota's universal free school meals program, which makes every MN student FREE-tier, so reduced-price eligibility is 0."
+us,scenario_032,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model compared roughly 156.7% FPL with the 185% reduced-price limit only. It missed Minnesota's universal free school meals program, which assigns the FREE tier and supersedes reduced-price eligibility."
+us,scenario_032,reduced_price_school_meals_eligible,inkling,llm_error,state_local_rule,False,"The model applied the federal $33,566–$47,767 reduced-price band. It never accounted for Minnesota's universal free school meals program, which puts the household in the FREE tier and makes reduced-price eligibility 0."
+us,scenario_032,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no usable value and no explanation for reduced_price_school_meals_eligible, so its answer was scored as missing. The correct value is 0, because Minnesota's universal free school meals program puts the household in the FREE tier."
+us,scenario_032,reduced_price_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model placed about 156% of poverty between the federal 130% and 185% limits and called it reduced-price. It missed Minnesota's universal free school meals program, which assigns the FREE tier regardless of income."
+us,scenario_032,reduced_price_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,"The model used only the federal 130–185% FPL band. It ignored that Minnesota provides universal free school meals, so the engine assigns the FREE tier and returns no reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"The model concluded reduced-price eligibility because income was above the free limit and below 185% of poverty. It overlooked Minnesota's universal free school meals program, which makes the household FREE-tier no matter what its income is."
+us,scenario_032,reduced_price_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model applied the federal 185% reduced-price cutoff to income of about 160% of poverty. It missed that Minnesota's universal free school meals program puts the child in the FREE tier, which excludes reduced-price support."
us,scenario_032,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_032,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly used Minnesota broad-based categorical eligibility to bypass the failed SNAP gross-income screen, then contradicted its own calculation showing that net income produced a negative allotment. It replaced that result with an unsupported $493 monthly benefit despite no listed shelter expense or other deduction capable of generating it."
us,scenario_032,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Minnesota broad-based categorical eligibility as sufficient to pass the household through the gross-income screen. Its own max-allotment-minus-30%-of-net-income calculation was negative, but it then invented a $476 monthly benefit; SNAP's minimum benefit does not convert a negative allotment for this ineligible three-person household into that amount."
@@ -3394,97 +3441,99 @@ us,scenario_032,state_income_tax_before_refundable_credits,glm-5.3,llm_error,sta
us,scenario_032,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model asserted that Minnesota AGI equals federal AGI ""with no modifications,"" discarding the Minnesota Social Security subtraction that removes the entire $1,916 of federally taxable benefits at this income, and it also omitted the Minnesota dependent exemption of roughly $5,300 for Child 1. Applying that subtraction against its own $30,699 standard deduction gives $29,000 − $30,699 < 0, so taxable income is zero and the tax is $0, not the $217 base it taxed at 5.35%."
us,scenario_032,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so the required key was absent from the submitted outputs object. This is a failure to deliver the answer under the output contract, not a substantive error in the Minnesota computation."
us,scenario_032,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model declared the full $13,664 of Social Security ""fully taxable"" for Minnesota purposes, contradicting the §86 provisional-income formula (provisional income $35,832 against the $32,000 joint base makes only $1,916 taxable) and the Minnesota Social Security subtraction that removes even that $1,916, and it used a $28,500 joint standard deduction below Minnesota's 2026 amount with no dependent exemption. Its own stated chain of $42,664 − $28,500 at 5.35% yields $758, not the $128 it submitted, so the figure is an unsupported approximation layered on an inflated income base; the correct $29,000 base falls below the deduction and yields $0."
-us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Assumed the working-family part pays about $925 for a filer with one child. Under the combined CWFC, this household receives a working-family component of 4% of earnings up to $9,690, or $387.60, and a child component of $1,800. The $2,337 is a guess that overshoots the correct $2,187.60."
-us,scenario_032,state_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Added a one-child Working Family Credit add-on of about $950 on top of a $360 base. The current CWFC pays a base working-family component of $387.60 for this household, with no add-on for its child under 18. Its child amount of about $1,790 is below the indexed figure of $1,800."
-us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"Stated that Minnesota has no refundable individual income tax credits. The Minnesota Child and Working Family Credit is fully refundable, and this MFJ household with a 6-year-old and AGI below the phase-out threshold receives $2,187.60."
-us,scenario_032,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"Did not apply Minnesota's refundable Child and Working Family Credit. The household qualifies through one child aged 6 and $29,000 of earnings, yielding $1,800 child credit plus $387.60 working-family credit."
-us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Used the structure correctly but left the child amount at the 2023 figure of $1,750. The 2026 child amount is $1,800. Its $410 working-family estimate exceeds the $387.60 component calculated as 4% of earnings up to $9,690."
-us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Counted only the un-indexed $1,750 child credit. It dropped the $387.60 working-family component of the same combined credit and did not apply the $1,800 child amount for 2026."
-us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"Put the phase-out start at about $31,000 and reduced the credit for it. The MFJ threshold is $35,000 indexed upward, which is above the $30,916 AGI, so the full indexed child and working-family amounts totaling $2,187.60 apply."
-us,scenario_032,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Applied the 2023 base amounts of $1,750 child credit and $350 working-family cap without 2026 inflation indexing. The indexed amounts, $1,800 and $387.60, total $2,187.60."
-us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Modeled the Working Family Credit as about 35% of the federal EITC. It also left out the $1,800 child credit entirely; the current combined credit is that child amount plus a working-family component of $387.60."
-us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"Concluded that refundable credits require rent, property tax, or education expenses. The household meets the Child and Working Family Credit's eligibility rules and qualifies for $2,187.60."
-us,scenario_032,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Estimated the Working Family Credit at about $1,900 for one child, following the old EITC-style schedule. Under the current CWFC this household receives a working-family component of 4% of earnings up to $9,690, or $387.60, and a child component of $1,800."
-us,scenario_032,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Computed the Working Family Credit as 45% of the federal EITC. The current working-family component for this household is 4% of earnings up to $9,690, or $387.60, and the child amount is $1,800."
-us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"Computed the Working Family Credit as 33% of the federal EITC. The current component for this household is 4% of earnings up to $9,690, or $387.60. It also over-indexed the child credit to $1,912 when the 2026 amount is $1,800."
-us,scenario_032,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Used a $1,633 Working Family Credit from the old EITC-based schedule and a $1,950 child credit. The current combined credit pays $1,800 for the child plus $387.60 working-family credit."
-us,scenario_032,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Calculated the Working Family Credit as 45% of the federal EITC. The current working-family component for this household is $387.60, and the 2026 child amount is $1,800."
-us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"Phased the child credit down to $983.60 even though the $30,916 AGI is below the indexed MFJ threshold of $35,000+, so no reduction applies. It also guessed a $550 working-family credit when this household's component is $387.60."
-us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"Concluded no Minnesota refundable credit applies. The household qualifies for the refundable Child and Working Family Credit of $2,187.60, made up of $1,800 child credit and $387.60 working-family credit."
-us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Doubled the working-family base to $700 by counting each spouse. The base component is 4% of the return's earnings up to $9,690, yielding $387.60. It also left the child amount at $1,750; the indexed amount is $1,800."
-us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Estimated the Working Family Credit at about $1,200, following an EITC-style schedule that pays more for children. The current working-family component for this household is $387.60, and the child component is $1,800."
-us,scenario_032,state_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It supplied a $600 total without a component calculation. The full CWFC pays $1,800 plus $387.60, or $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"Counted only an un-indexed $1,750 child credit. It dropped the $387.60 working-family component and did not apply the $1,800 child amount for 2026."
-us,scenario_032,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Added a working-family amount of about $1,085 on top of $1,750. The current working-family component for this household is 4% of earnings up to $9,690, or $387.60, and the 2026 child amount is $1,800."
-us,scenario_032,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $2,450 total matches $1,750 plus a doubled $700 working-family base. The base working-family cap is $387.60 per return, and the child amount is indexed to $1,800."
-us,scenario_032,state_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"Its $2,110 total is close to the un-indexed 2023 amounts ($1,750 + $350), and it described a phase-out reduction that does not apply below the indexed MFJ threshold. The 2026 indexed amounts of $1,800 and $387.60 total $2,187.60."
-us,scenario_032,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Applied only the child component, indexed at an assumed 3% a year to $1,912, and said no other refundable credit applies. That over-indexed the $1,800 child amount and left out the $387.60 working-family component."
-us,scenario_032,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"Used the $29,500 single/head-of-household phase-out threshold for this joint return. The indexed MFJ threshold is $38,770, which is above the $30,916 AGI, so the full $1,800 child credit and $387.60 working-family credit apply."
-us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"Did not find Minnesota's refundable Child and Working Family Credit. The household earns $29,000 and has a qualifying 6-year-old, which yields $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Defaulted to zero because no credit amounts were listed in the facts. The refundable CWFC is computed from earnings and the qualifying child, giving $1,800 plus $387.60."
-us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Used the 2023 base amounts of $1,750 and $350 without 2026 inflation indexing. The indexed child amount of $1,800 and base working-family cap of $387.60 give $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Computed the Working Family Credit as 25% of the federal EITC. The current component for this household is 4% of earnings up to $9,690, or $387.60, and the child amount is $1,800."
-us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Used a $1,675 Working Family Credit from the old EITC-style one-child schedule. The current working-family component for this household is $387.60, and the child amount is $1,800."
-us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $3,572 total implies a Working Family Credit near $1,800 under the repealed EITC-based schedule. The current CWFC pays $1,800 for the child plus a working-family component of $387.60."
-us,scenario_032,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"Left the child credit at $1,750; the 2026 amount is $1,800. Its $388 working-family estimate rounds the $387.60 component to the nearest dollar."
-us,scenario_032,state_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Its $4,500 total implies a Working Family Credit of roughly $2,700 under an old EITC-style schedule. The current CWFC pays $1,800 for the child plus $387.60 working-family credit."
-us,scenario_032,state_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"Used a $1,275 Working Family Credit from the old child-based schedule. The current working-family component for this household is 4% of earnings up to $9,690, or $387.60, and the child amount is $1,800."
-us,scenario_032,state_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"Left the child credit at $1,750; the 2026 amount is $1,800. Its $387 working-family estimate is $0.60 below the $387.60 component."
-us,scenario_032,state_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"Its $3,150 total implies a Working Family Credit of about $1,400 under the repealed EITC-based schedule. The current working-family component for this household is $387.60, and the child amount is $1,800."
-us,scenario_032,state_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Counted only a Working Family Credit of about $500. It left out the $1,800 child component, which is the larger part of the combined CWFC."
-us,scenario_032,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Computed the Working Family Credit as 40% of the federal EITC. The current component for this household is $387.60. It also left the child credit at $1,750; the indexed amount is $1,800."
-us,scenario_032,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Estimated the Working Family Credit at $1,560 under the old EITC-style schedule. The current component is $387.60, and the 2026 child amount is $1,800."
-us,scenario_032,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Used a one-child Working Family Credit maximum of $1,577. The combined CWFC gives this household a base working-family component of $387.60 alongside the $1,800 child credit, with no working-family add-on for its child under 18."
-us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Computed the Working Family Credit as 33% of the federal EITC. The current component for this household is $387.60. It also left the child credit at $1,750; the indexed amount is $1,800."
-us,scenario_032,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"Estimated a one-child Working Family Credit of about $1,300 under the old schedule. The current working-family component for this household is 4% of earnings up to $9,690, or $387.60, and the child amount is $1,800."
-us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable value or explanation for state_refundable_credits, so no answer could be scored against the $2,187.60 reference."
-us,scenario_032,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Used a full one-child Working Family Credit of $1,250 from the old schedule. The current working-family component is $387.60, and the 2026 child amount is $1,800."
-us,scenario_032,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"Denied the credit, saying household income is above the eligibility thresholds and the spouse has no earnings. The CWFC needs earnings from only one spouse, and the $30,916 AGI is below the indexed MFJ phase-out start, so the full $2,187.60 is payable."
-us,scenario_032,state_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"Estimated the Working Family Credit at about $1,100 under an EITC-style schedule. The current working-family component for this household is $387.60, and the indexed child amount is $1,800."
-us,scenario_032,state_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"Estimated the Working Family Credit at about $870. The current component for this household is 4% of earnings up to $9,690, or $387.60, and the 2026 child amount is $1,800."
-us,scenario_032,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Started from a roughly $2,900 Working Family Credit maximum and applied a phase-out to reach $1,650. The current working-family component for this household is $387.60 with no phase-out at this AGI, and the child amount is $1,800."
-us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Computed only a Working Family Credit from federal EITC percentages. It left out the $1,800 child component and overstated the working-family part, which is $387.60 for this household."
-us,scenario_032,state_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Used a $1,648 EITC-style Working Family Credit and added a $241 dependent-care credit even though there are no care expenses. It also left out the $1,800 child component; the correct total is $1,800 plus $387.60."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $243 state tax refund out of income, which put AGI at $83,208 and taxable income at $35,708 instead of $35,951.23. It then computed $3,789 on its own taxable income but submitted an unexplained $3,699."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI ($83,451), standard deduction ($35,500) and senior deduction ($12,000) all match the reference. However, it also subtracted $1,225 of auto-loan interest as a qualified vehicle-loan interest deduction, even though no qualifying new-vehicle purchase is given. That cut taxable income to $34,726 instead of $35,951 and the tax to $3,671 instead of $3,818."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the standard deduction inside AGI and used a $30,400 standard deduction with no aged additions and no $12,000 senior deduction. It also treated the $11,400 business loss as a capital loss capped at $3,000. Its final $5,661 doesn't follow from any of its own intermediate figures."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. The correct taxable income is $35,951 ($83,451 AGI minus $47,500 of deductions), which gives $3,818. Its $2,050 implies roughly $20,500 of taxable income, so it overstated deductions or understated taxable Social Security by about $15,000."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $243 state refund (AGI $83,208) and ended the 10% bracket at $24,150 instead of the 2026 MFJ figure of $24,800. It computed $3,801.96, then submitted an arbitrary $3,416."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It added non-Social Security income as $67,618 instead of $68,618, a $1,000 arithmetic slip. It used the 2025 base standard deduction ($31,500 instead of $32,200) and ended the 10% bracket at $24,000 instead of $24,800. It reached about $3,798 and then submitted an unrelated $4,575."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It added up its pre-Social Security AGI as $66,131 instead of $67,131, understating taxable income by $1,000 ($34,951 instead of $35,951). It then added $2,480 + $1,218 and got $3,080 instead of $3,698."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 is correct. It left out the OBBBA $6,000-per-senior deduction and used 2025 figures ($30,000 base and $1,600 aged additions instead of $32,200 and $1,650). That left taxable income at $50,251 instead of $35,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped the $243 refund, used $1,600 aged additions, and never applied the $12,000 senior deduction, so taxable income came out at $47,808. It computed $5,241 and then submitted an unsupported $3,800."
-us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 is correct. It used a $31,000 base standard deduction instead of $32,200 and left out the $6,000-per-senior OBBBA deduction, so it taxed $49,151 instead of $35,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 2024 rules: a $33,100 standard deduction and a 10% bracket ending at $23,200. It left out the 2026 $32,200 base, the $1,650 aged additions and the $12,000 senior deduction, so it taxed $50,351 instead of $35,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA expires in 2026 and applied a pre-TCJA $16,700 standard deduction, personal exemptions and a 15% bracket. OBBBA made the TCJA standard deduction and rates permanent and added the $6,000-per-senior deduction, so the correct taxable income is $35,951, taxed at 10%/12%."
-us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law: a $16,600 standard deduction, $10,600 of personal exemptions and a 15% second bracket. It missed that OBBBA kept the $32,200 MFJ standard deduction and the 12% bracket and added the $12,000 senior deduction for this couple."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026, using a $16,000 standard deduction, $10,000 of personal exemptions and a 15% bracket, and it also dropped the $243 refund. Under OBBBA, a $47,500 deduction (standard + aged + senior) applies and the second bracket is 12%."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It understated AGI as $78,818 instead of $83,451 and used a $32,500 deduction, leaving out the $3,300 aged additions and the $12,000 senior deduction. Its $3,724 doesn't even match its own $46,318 taxable income at 10%/12%."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. Its $6,762 matches the TCJA-sunset calculations other models made: about $53,000 of taxable income after pre-TCJA deductions and exemptions, taxed at 10%/15%. The OBBBA $47,500 deduction stack and 12% bracket give $3,818."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026, applying a $16,500 standard deduction, $10,500 of personal exemptions and reverted brackets, and it also dropped the $243 refund. OBBBA's permanent $32,200 standard deduction, the $3,300 aged additions, the $12,000 senior deduction and the 12% bracket apply instead."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed deductions and credits wipe out the tax. But $83,451 of AGI less the $47,500 standard, aged and senior deductions leaves $35,951 of taxable income, and no nonrefundable credit applies, so the tax is $3,818, not $0."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped the $243 refund (AGI $83,208), used a $34,000 standard deduction, and never applied the OBBBA $6,000-per-senior deduction, so it taxed $49,208 instead of $35,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped the $243 refund and applied personal exemptions, which OBBBA permanently eliminated, instead of the $12,000 senior deduction. That produced $5,045 instead of tax on $35,951 of taxable income."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 is correct. It used 2025 amounts ($30,000 base, $1,600 aged additions, 10% bracket ending at $23,850) and left out the $6,000-per-senior deduction, so it taxed $50,251 instead of $35,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $11,400 SSTB self-employment loss out of AGI ($94,851 instead of $83,451). It also subtracted $1,225 of auto-loan interest, which isn't a deductible item here. Together these errors put taxable income at $46,126 instead of $35,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 is correct. It underestimated the standard deduction as $34,030 instead of $35,500 and left out the OBBBA $6,000-per-senior deduction for both spouses, so it taxed $49,421 instead of $35,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It misapplied the Social Security taxability formula, getting $12,085 taxable instead of $16,320. With provisional income far above $44,000, 85% of benefits is taxable. It also left out the $1,650 aged addition for each spouse and ended the 10% bracket at $24,500 instead of $24,800."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It cited itemized mortgage-interest and SALT amounts even though no mortgage interest or deductible taxes are given; only a loan balance is listed. Its $3,677 implies about $34,775 of taxable income, roughly $1,176 below the correct $35,951 (AGI $83,451 less the $47,500 standard, aged and senior deductions)."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It treated Social Security as non-taxable. With provisional income around $76,700, $16,320 (85%) is taxable, giving $83,451 of AGI and $35,951 of taxable income, so the tax is $3,818, not $0."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-TCJA-sunset law: $29,900 of standard/age deductions plus personal exemptions, with a 15% second bracket. It missed OBBBA's permanent $32,200 standard deduction, the 12% bracket and the $12,000 senior deduction."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Everything else matches the reference, but it added a $333.33 qualified-overtime deduction by inferring overtime from $53,000 of wages versus $25/hour × 40 hours × 52 weeks. No overtime income is listed, so that deduction is zero, and taxable income stays at $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. Its $5,523 implies about $49,500 of taxable income, which is what results from leaving out the $12,000 OBBBA senior deduction; the correct taxable income is $35,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 is correct. It applied only a $34,030 standard deduction and left out the $6,000-per-senior deduction for both spouses, so it taxed $49,421 instead of $35,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law: a $16,600 standard deduction, $5,300 personal exemptions and a 15% bracket. OBBBA made the $32,200 standard deduction and 12% bracket permanent and added the $12,000 senior deduction."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $243 state refund as non-taxable, which put AGI at $83,208 and taxable income at $35,708 instead of $35,951. It also ended the 10% bracket at $24,400 instead of the 2026 MFJ figure of $24,800."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 is correct. It used a $34,196 standard deduction and left out the OBBBA $6,000-per-senior deduction, so it taxed $49,255 instead of $35,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $1,225 of auto-loan interest above the line and added a nonexistent 'temporary $1,500' to the standard deduction ($37,000 instead of $35,500). Those errors cut taxable income to $33,226 instead of $35,951."
-us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was no answer to score."
-us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI and the $47,500 standard, aged and senior deductions match the reference. It then also subtracted $1,225 of auto-loan interest and a $333.33 overtime deduction inferred from the hourly wage; neither applies. That dropped taxable income to $34,392.67 instead of $35,951.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $11,400 SE loss, mixed in fully taxable Social Security, and left out the $12,000 senior deduction, reaching about $10,800. It then submitted −$250, a negative number that is impossible for tax before refundable credits."
-us,scenario_033,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"Its method matches the reference ($83,451 AGI less $47,500 of deductions). But it rounded taxable income to about $36,000 instead of carrying the exact $35,951.23, so it submitted $3,820 instead of $3,818.15."
-us,scenario_033,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $11,400 SSTB self-employment loss out of AGI ($94,608 instead of $83,451) and left out the $12,000 senior deduction. It computed $6,748 and then submitted an unsupported $5,998."
-us,scenario_033,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard deduction and a Social Security 'subtraction' zero out taxable income. Federal law includes 85% of benefits ($16,320) here, leaving $35,951 of taxable income and $3,818 of tax."
+us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model valued the working-family part at about $925 for a filer with one child. Under the CWFC, that part is a per-return amount of 4% of earnings capped near $365, with no per-child increase. It also used the unindexed $1,750 child credit and then made an arbitrary reduction, landing on $2,337 instead of $2,187.60."
+us,scenario_032,state_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The model invented a one-child add-on of about $950 to the working family credit. Under the CWFC, the only per-child amount is the indexed child credit (about $1,823), and the working-family part is a flat 4%-of-earnings amount capped near $365. Stacking a third component pushed the total to $3,100."
+us,scenario_032,state_refundable_credits,claude-fable-5.2,llm_error,thresholds_rates,False,"The model left the child credit at the unindexed $1,750 instead of the 2026 indexed amount of about $1,823. It also set the working-family cap at $388 instead of about $365. The result, $2,138, is $49.60 short of $2,187.60."
+us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model claimed Minnesota has no refundable individual credits. It missed the fully refundable Child and Working Family Credit, which this married couple with a 6-year-old and $29,000 of earnings qualifies for at $2,187.60."
+us,scenario_032,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model found no Minnesota refundable credit. It missed the refundable Child and Working Family Credit, which pays the indexed child credit plus the capped 4%-of-earnings working-family part, $2,187.60 in total, because AGI of $30,916 is below the joint phaseout threshold."
+us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the unindexed $1,750 child credit and put the working-family cap at $410. The 2026 indexed values are about $1,823 and about $365, which sum to $2,187.60, so its $2,160 understates the child part and overstates the working-family part."
+us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model counted only an unindexed $1,750 child credit and dropped the working-family part. That part is 4% of $29,000 of earnings, capped at about $365. Indexing the child credit to about $1,823 and adding the working-family part gives $2,187.60."
+us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model assumed a shared phaseout starting around $31,000 and trimmed the credit. The 2026 joint threshold is about $36,460, so no phaseout applies at $30,916 of AGI. It also started from the unindexed $1,750 child credit, which together with the phaseout trim left its $2,185 short of the full $2,187.60."
+us,scenario_032,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model used the base amounts, a $1,750 child credit and a $350 working-family cap, without 2026 inflation indexing. Indexing raises both by about 4.17%, to roughly $1,823 and $365, for $2,187.60 instead of $2,100."
+us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied the repealed WFC formula, a percentage of the federal EITC, and omitted the CWFC child credit entirely. It also put 85% of Social Security into AGI, but only $1,916 is taxable, so AGI is $30,916. The correct CWFC is the indexed child credit (about $1,823) plus the 4% working-family part capped at about $365, or $2,187.60."
+us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model assumed Minnesota's refundable credits require rent, education expenses, or extra verified inputs, and returned 0. The Child and Working Family Credit needs only a qualifying child and earned income, and pays $2,187.60 here."
+us,scenario_032,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The model estimated a $1,900 working family credit, which reflects the repealed EITC-based WFC. Under the CWFC, the working-family part is only 4% of earnings capped near $365, added to an indexed child credit of about $1,823, for $2,187.60 instead of $3,700."
+us,scenario_032,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model computed a separate WFC as 45% of the federal EITC ($2,081) and stacked it on a $1,750 child credit. The EITC-percentage WFC was replaced by the CWFC, whose working-family part is only 4% of earnings capped at about $365, so the total is $2,187.60."
+us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model added a WFC equal to 33% of the federal EITC ($1,481), which is the repealed structure, to an over-indexed $1,912 child credit. Under the CWFC, the working-family part is capped at about $365 and the child credit is about $1,823, for $2,187.60."
+us,scenario_032,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model valued the working family credit at $1,633, an EITC-style amount, and the child credit at $1,950. Under the 2026 CWFC, the working-family part is 4% of earnings capped at about $365 and the child credit is about $1,823, for $2,187.60."
+us,scenario_032,state_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model computed the WFC as 45% of the federal EITC ($1,368) on top of a $1,750 child credit, using the pre-2023 structure. The CWFC working-family part is 4% of earnings capped at about $365, and the indexed child credit is about $1,823, for $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model applied a phaseout that cut the child credit to $983.60, even though AGI of $30,916 is below the joint threshold (about $36,460). It also put the working-family part at $550 instead of the roughly $365 cap. With no phaseout, the CWFC is $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model concluded that no Minnesota refundable credit applies. This low-income married couple with a 6-year-old qualifies for the refundable Child and Working Family Credit of $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $700 working-family amount for the couple, treating it as a per-adult benefit. That part is a single per-return amount of 4% of earnings capped at about $365. The model also left the child credit at the unindexed $1,750 instead of about $1,823."
+us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model estimated about $1,200 for the working family credit. The CWFC working-family part is capped at about $365 (4% of earnings up to the cap), and the indexed child credit is about $1,823 rather than $1,750, so the total is $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The $600 answer is consistent with counting only a small working-family amount and omitting the per-child credit. The CWFC pays an indexed child credit of about $1,823 for the 6-year-old plus a working-family part of about $365, for $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model counted only an unindexed $1,750 child credit. It omitted the CWFC working-family part (4% of earnings, capped at about $365) and the 2026 indexing that raises the child credit to about $1,823."
+us,scenario_032,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's total implies a working family credit of about $1,085 on top of $1,750, which is an EITC-style amount. The CWFC working-family part is capped at about $365, and the indexed child credit is about $1,823, for $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The $2,450 total implies a working-family amount of about $700 on top of a $1,750 child credit. The CWFC working-family part is 4% of earnings capped at about $365, and the indexed child credit is about $1,823, for $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"The model said a phaseout reduced the credit, but AGI of $30,916 is below the indexed joint threshold, so none applies. Its $2,110 matches roughly unindexed component values ($1,750 plus about $360), not the 2026 indexed $1,823 plus $365."
+us,scenario_032,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model indexed the child credit to about $1,912 but explicitly said no other refundable credit applies. That dropped the CWFC working-family part, 4% of $29,000 capped at about $365, which is paid alongside the indexed child credit of about $1,823."
+us,scenario_032,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model applied the $29,500 phaseout threshold, which is for single and head-of-household filers, to this joint return. That wrongly cut the child credit and shrank the working-family part. The joint threshold (about $36,460 in 2026) is above $30,916 of AGI, so the full $2,187.60 is paid."
+us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model found no Minnesota refundable credit. It missed the Child and Working Family Credit, which pays $2,187.60 to this joint filer with one child and $29,000 of earnings."
+us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the refundable credit as requiring a separately listed eligibility amount and returned 0. The Child and Working Family Credit is computed directly from the qualifying child and earnings, giving $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used the 2024 base values, a $1,750 child credit and a $350 working-family cap, without 2026 inflation indexing. The indexed values (about $1,823 and $365) total $2,187.60, not $2,100."
+us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model computed the working family credit as 25% of the federal EITC ($1,106.75), which is the repealed structure, and added it to a $1,750 child credit. The CWFC working-family part is 4% of earnings capped at about $365, so the total is $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model put the Working Family Credit at $1,675, an EITC-style amount. Under the CWFC, that part is capped at about $365, and with the indexed child credit of about $1,823 the total is $2,187.60, not $3,425."
+us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The $3,572 total implies a working family credit of roughly $1,800 on top of the child credit, which reflects the repealed EITC-based WFC. The CWFC working-family part is capped at about $365, giving $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"The model kept the child credit at the unindexed $1,750 instead of the 2026 amount of about $1,823. It also used a $388 working-family cap instead of about $365, so it was $49.60 short."
+us,scenario_032,state_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"The model's $4,500 grossly overstates the working-family part, treating it like a large EITC-style credit. The CWFC working-family part is 4% of earnings capped at about $365, and with the indexed child credit of about $1,823 the total is $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"The model used a $1,275 working family credit, which reflects the pre-2023 EITC-based WFC. The CWFC working-family part is capped at about $365, and the child credit is indexed to about $1,823, for $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"The model used the unindexed $1,750 child credit instead of the 2026 amount of about $1,823. It also used a $387 working-family cap instead of about $365, so it was $50.60 short of $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"The $3,150 total implies a working family credit of about $1,400 on top of a $1,750 child credit, which is an EITC-style amount. The CWFC working-family part is capped at about $365, giving $2,187.60."
+us,scenario_032,state_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model counted only a WFC-like amount of $500 and omitted the CWFC per-child credit. For the 6-year-old, that child credit is about $1,823 and makes up most of the $2,187.60."
+us,scenario_032,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model computed the WFC as 40% of the federal EITC ($1,774), which is the repealed structure. The CWFC working-family part is 4% of earnings capped at about $365, and with the indexed child credit of about $1,823 the total is $2,187.60."
+us,scenario_032,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model estimated a $1,560 working family credit, an EITC-style figure. Under the CWFC, the working-family part is capped at about $365, and with the indexed child credit of about $1,823 the total is $2,187.60."
+us,scenario_032,state_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model used $1,577 as the one-child maximum for the working family credit, carried over from the old EITC-based WFC schedule. The CWFC working-family part has no per-child maximum; it is capped at about $365, for a total of $2,187.60."
+us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model computed the WFC as 33% of the federal EITC ($1,465), which is the repealed structure. The CWFC working-family part is 4% of earnings capped at about $365, and the child credit is indexed to about $1,823, for $2,187.60."
+us,scenario_032,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model estimated a $1,300 one-child working family credit, an EITC-style amount. The CWFC working-family part is capped at about $365 regardless of the number of children, so the total is $2,187.60."
+us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value for state_refundable_credits, so there was nothing to score against the $2,187.60 CWFC."
+us,scenario_032,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used a $1,250 one-child working family credit, which reflects the old EITC-style schedule. The CWFC working-family part is capped at about $365, and the indexed child credit is about $1,823, for $2,187.60."
+us,scenario_032,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model claimed the household was ineligible because the spouse has no earnings and income is too high. Only one filer needs earned income, and AGI of $30,916 is below the joint phaseout threshold, so the full $2,187.60 CWFC is paid."
+us,scenario_032,state_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"The model estimated about $1,100 for the working family credit, an EITC-style amount. The CWFC working-family part is capped at about $365, and the indexed child credit is about $1,823, for $2,187.60."
+us,scenario_032,state_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model estimated about $870 for the working family credit. The CWFC working-family part is 4% of earnings capped at about $365, and the child credit is indexed to about $1,823, for $2,187.60."
+us,scenario_032,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model assumed a working family credit maximum of about $2,900, reduced to $1,650 by a 12% phaseout. The CWFC working-family part is capped at about $365, and no phaseout applies below the roughly $36,460 joint threshold, so the total is $2,187.60."
+us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model computed only an EITC-percentage WFC of $1,895 and omitted the per-child credit. The CWFC is an indexed child credit of about $1,823 plus a working-family part capped at about $365, for $2,187.60."
+us,scenario_032,state_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used an EITC-style $1,648 working family credit and added a $241 dependent care credit even though there are no care expenses. It also omitted the CWFC child credit (about $1,823), which together with the roughly $365 working-family part gives $2,187.60."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $243 state tax refund out of gross income, so AGI came to $83,208 instead of $83,451.23 and taxable income to $35,708 instead of $35,951.23. It then correctly worked out $3,789 from its own figures but reported $3,699, an unexplained 'rounding' change that contradicts its own arithmetic."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted $1,225 of auto-loan interest as an OBBBA vehicle-loan interest deduction. No fact establishes a qualifying new, US-assembled vehicle, so the deduction doesn't apply. Without it, taxable income is $35,951.23 rather than $34,726, and tax is $3,818.15."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI, standard deduction and senior deduction were all correct, but it also subtracted a $1,225 auto-loan interest deduction that no qualifying-vehicle fact supports. That cut taxable income to $34,726 instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the $11,400 SSTB business loss as a capital loss capped at $3,000, when the full business loss offsets income. It also subtracted the standard deduction inside AGI using an outdated $30,400 figure, with no $3,300 age additions and no $12,000 senior deduction. It then reported $5,661, which doesn't follow from its own $6,959 figure."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. $2,050 implies taxable income of about $20,500, roughly $15,000 below the correct $35,951.23 (AGI of $83,451.23 minus $47,500 of standard plus senior deductions), so it over-deducted or understated taxable Social Security."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $243 state refund (AGI $83,208) and used a 10% bracket ending at $24,150 instead of the 2026 MFJ $24,800. It computed $3,801.96 and then reported $3,416 with no supporting calculation."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It made an addition error: $79,000 − $11,400 + $775 + $243 is $68,618, not $67,618, which put AGI $1,000 too low. It also used a $31,500 base standard deduction instead of $32,200 and a $24,000 top for the 10% bracket instead of $24,800. It then reported $4,575, which contradicts its own $3,798 figure."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It made two arithmetic errors. Pre-Social Security income is $67,131, not $66,131, which left taxable income at $34,951 instead of $35,951.23. Then $2,480 + $1,218 is $3,698, not $3,080."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It used the pre-OBBBA 2025 standard deduction ($30,000 + 2×$1,600) and left out the $6,000-per-senior OBBBA deduction for both spouses. That inflated taxable income to $50,251 instead of $35,951.23. It also used an estimated $24,500 top for the 10% bracket instead of $24,800."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $243 refund, used $1,600 instead of $1,650 for each age addition, and omitted the $12,000 senior deduction. That produced $5,241 of tax, which it then cut to $3,800 for unspecified 'adjustments' that its reasoning never supports."
+us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it used a $31,000 base standard deduction instead of $32,200 and left out the $12,000 senior deduction for two spouses aged 65+. That overstated taxable income at $49,151 instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 2024 rules, with a $33,100 standard deduction and a $23,200 top for the 10% bracket, instead of the 2026 figures ($35,500 standard deduction, $24,800 bracket). It also left out the $12,000 OBBBA senior deduction, so taxable income came to $50,351 instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired in 2026 and used pre-TCJA personal exemptions, a $16,700 standard deduction and a 15% bracket. OBBBA made the TCJA rate structure permanent, so 2026 has no personal exemptions, a $35,500 standard deduction plus the $12,000 senior deduction, and a 12% bracket."
+us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied a TCJA sunset (personal exemptions of $10,600, a $16,600 standard deduction, a 15% bracket) even though OBBBA extended the TCJA structure. The 2026 computation uses $47,500 of standard plus senior deductions and 10%/12% brackets."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired, using personal exemptions, a $16,000 standard deduction and a 15% bracket, instead of the OBBBA-extended 2026 law with a $35,500 standard deduction, the $12,000 senior deduction and a 12% bracket. It also left out the $243 refund from AGI."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It understated AGI at $78,818 instead of $83,451.23, used a $32,500 standard deduction without the $3,300 age additions, and left out the $12,000 senior deduction. Its $3,724 also doesn't match its own $46,318 taxable income, which would produce about $5,060."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. $6,762 matches a TCJA-sunset calculation (personal exemptions, a reduced standard deduction, a 15% bracket on about $53,000 of taxable income), not the OBBBA 2026 calculation of $35,951.23 taxable income at 10%/12%."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly assumed the TCJA had expired, using a $16,500 standard deduction, personal exemptions and 'reverted' brackets. OBBBA kept the TCJA structure, so 2026 has a $35,500 standard deduction plus the $12,000 senior deduction and 10%/12% brackets. It also left out the $243 refund from AGI."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed deductions, exemptions and credits wipe out the tax. In fact AGI of $83,451.23 minus $47,500 of standard plus senior deductions leaves $35,951.23 of taxable income, and there are no nonrefundable credits to offset the $3,818.15 tax."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $243 refund from AGI, used a $34,000 standard deduction instead of $35,500, and omitted the $12,000 senior deduction. That gave $49,208 of taxable income instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $243 refund from AGI and subtracted personal exemptions, which are $0 under the OBBBA-extended TCJA rules for 2026. It also never applied the $12,000 senior deduction, so its $5,045 overstates the tax."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used the 2025 pre-OBBBA figures ($30,000 + 2×$1,600 standard deduction, a $23,850 top for the 10% bracket) and left out the $12,000 senior deduction. Taxable income came to $50,251 instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"It never subtracted the $11,400 SSTB self-employment loss, so AGI came to $94,851 instead of $83,451.23. It also took a $1,225 auto-loan interest deduction that no qualifying-vehicle fact supports."
+us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI was correct, but it used a $34,030 standard deduction instead of $35,500 and omitted the OBBBA $6,000-per-senior deduction for both spouses. That gave $49,421 of taxable income instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used a non-existent $64,000 threshold in the taxable Social Security formula, getting $12,085 taxable instead of the 85% cap of $16,320. It also dropped the 2×$1,650 age additions from the standard deduction. Both errors moved taxable income away from $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It cited mortgage-interest itemizing even though no mortgage interest is listed. Its $3,677 implies taxable income of about $34,775, roughly $1,176 below the correct $35,951.23 (AGI $83,451.23 minus $47,500 of standard plus senior deductions), which matches taking a spurious extra deduction."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It treated Social Security as non-taxable and concluded the tax was zero. Provisional income far exceeds $44,000, so $16,320 of benefits is taxable, AGI is $83,451.23, and taxable income after $47,500 of deductions is $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it applied 'post-TCJA' 2026 rules with personal exemptions and a 15% bracket. OBBBA kept the TCJA structure, so 2026 has a $35,500 standard deduction plus the $12,000 senior deduction and a 12% bracket."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"From the gap between $53,000 of wages and $25/hour × 40 hours × 52 weeks, it inferred $1,000 of overtime and took a $333.33 overtime-premium deduction. The facts list no overtime income, so no overtime deduction applies and taxable income is $35,951.23, not $35,617.67."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no detailed derivation. $5,523 matches roughly $50,000 of taxable income, meaning it subtracted only the standard deduction and omitted the $12,000 OBBBA senior deduction that brings taxable income down to $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it used a $34,030 projected standard deduction instead of $35,500 and left out the $12,000 senior deduction. That gave $49,421 of taxable income instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It applied 'sunset' law ($16,600 standard deduction, $5,300 personal exemptions, a 15% bracket) even though OBBBA made the TCJA structure permanent. 2026 tax uses $47,500 of standard plus senior deductions and 10%/12% brackets."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $243 state refund (AGI $83,208, taxable income $35,708) and used a $24,400 top for the 10% bracket instead of the 2026 MFJ $24,800."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Its AGI of $83,451 was correct, but it used a $34,196 standard deduction instead of $35,500 and omitted the $12,000 OBBBA senior deduction. That gave $49,255 of taxable income instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $1,225 of auto-loan interest even though no qualifying-vehicle fact supports it, and added a non-existent 'temporary $1,500' to the standard deduction (making it $37,000 instead of $35,500). That gave $33,226 of taxable income instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no answer and no explanation, so there was no value to score."
+us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $1,225 auto-loan interest deduction and a $333.33 overtime-premium deduction inferred from the hourly wage, though the facts support neither. Taxable income came to $34,392.67 instead of $35,951.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It never subtracted the $11,400 SSTB loss and omitted the $12,000 senior deduction, which got it to about $10,800 of tax. It then reported −$250, a negative amount for tax before refundable credits that contradicts its own computation."
+us,scenario_033,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"It had the right structure (AGI $83,451, $47,500 of deductions) but approximated taxable income as '~$36k' and rounded the result to $3,820. Exact taxable income is $35,951.23, which gives $2,480 + 12% × $11,151.23 = $3,818.15."
+us,scenario_033,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It dropped the $11,400 SSTB loss from AGI (getting $94,608) and omitted the $12,000 senior deduction. It then reported $5,998, which contradicts its own $6,748 calculation."
+us,scenario_033,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed taxable income was zero, but AGI of $83,451.23 (including $16,320 of taxable Social Security) minus $47,500 of standard plus senior deductions leaves $35,951.23 taxable. There are no nonrefundable credits to offset the resulting $3,818.15."
us,scenario_033,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_033,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It correctly computed $1,612 + $377 + $3,286 + $768.50 = $6,043.50. It then added a made-up $64.15 'rounding adjustment' that has no basis in any payroll tax rule, and submitted $6,107.65."
us,scenario_033,payroll_tax,claude-haiku-5.5,llm_error,other,False,"It correctly computed $4,054.50 + $1,989 = $6,043.50 in FICA. It then stated 6044 and submitted 6045, rounding up for no reason instead of reporting the exact $6,043.50."
@@ -3612,21 +3661,22 @@ us,scenario_037,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_err
us,scenario_037,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable output for federal_refundable_credits and therefore failed the required submission contract.
us,scenario_037,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented dependent-child credit eligibility despite the household containing only one 44-year-old adult: refundable CTC is $0 because there is no qualifying child. It also applied a child-based EITC amount even though the filer has no qualifying children, and $41,943 exceeds the childless EITC income limit, so EITC is $0."
us,scenario_037,head_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model treated North Carolina’s adoption of Medicaid expansion as sufficient for eligibility and failed to compare the head’s MAGI income of 2.60 times FPL with the expansion-adult income limit. The head exceeds that limit and has no alternative Medicaid pathway, yielding medicaid_category NONE and ineligibility."
-us,scenario_037,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly computed 6.2% and 1.45% of $41,943 and got a total of $3,208.74. It then swapped in $3,211.64 for unexplained 'minor adjustments,' adding about $3 that no payroll tax rule supports."
-us,scenario_037,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"The model named the right rates (6.2% Social Security and 1.45% Medicare on gross wages) but submitted 0 as a placeholder and never applied them. Applying 7.65% to the $41,943 of wages gives about $3,208.67."
-us,scenario_037,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the $5,789 of employer-sponsored insurance premiums from wages and used a FICA base of $36,154. Nothing in the facts says the premiums are paid through a pre-tax cafeteria plan, so FICA applies to the full $41,943, which gives $3,208.67."
-us,scenario_037,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated the $5,789 of ESI premiums as a pre-tax payroll deduction and taxed only $36,154. That assumes a Section 125 salary-reduction election the household facts don't list. Social Security and Medicare apply to the full $41,943 of gross wages."
-us,scenario_037,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model left the $5,789 of employer-sponsored insurance premiums out of FICA wages and applied 7.65% to $36,154. The payroll tax base is the full $41,943 of gross wages, so Social Security is $2,600.49 and Medicare is $608.18."
-us,scenario_037,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model cut the FICA base to $36,154 by treating ESI premiums as pre-tax. The facts include no cafeteria-plan salary reduction, so 6.2% and 1.45% apply to the full $41,943, for a total of $3,208.67."
-us,scenario_037,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model netted the $5,789 of ESI premiums out of wages before applying 7.65%, understating the base by $5,789. Employee FICA applies to all $41,943 of gross wages."
-us,scenario_037,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly reached about $2,601 + $608 = $3,209 on the full $41,943 of wages. It then reported $3,151 with no rule behind the $58 cut, which contradicts its own arithmetic."
-us,scenario_037,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model correctly kept traditional 401(k) contributions in FICA wages but wrongly took out the $5,789 of employer-sponsored insurance premiums, taxing only $36,154. No pre-tax premium arrangement is listed, so the base is the full $41,943, which gives $3,208.67."
-us,scenario_037,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model applied 7.65% to $36,154 after taking out the $5,789 of ESI premiums, assuming a Section 125 salary reduction the facts don't include. The FICA base is the full $41,943 of gross wages."
-us,scenario_037,payroll_tax,grok-4.3,llm_error,other,False,"The model used the right method, the employee share of Social Security and Medicare on gross wages, but reported a rounded $3,210 instead of the exact amount. Exactly, 6.2% gives $2,600.49 and 1.45% gives $608.18, for $3,208.67, so the answer overshoots by $1.33."
-us,scenario_037,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted the $5,789 of ESI premiums as pre-tax, taxed $36,154, and rounded each piece to whole dollars. Payroll tax applies to the full $41,943 of gross wages, which gives $3,208.67."
-us,scenario_037,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model treated the $5,789 of employer-sponsored health premiums as excluded from FICA wages and computed tax on $36,154. Without a listed cafeteria-plan salary reduction, Social Security and Medicare apply to the full $41,943."
-us,scenario_037,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no payroll_tax value and no reasoning, so its answer couldn't be parsed. The correct value is 7.65% of $41,943 in gross wages, or $3,208.67."
-us,scenario_037,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly computed about $2,601 in Social Security and $608 in Medicare on the full $41,943, but added them to $3,213 instead of $3,209. That is an arithmetic slip on top of a correct method."
+us,scenario_037,payroll_tax,claude-fable-5.2,llm_error,payroll_tax_base,False,"Subtracted the $386 traditional 401(k) deferral from FICA wages, using a $41,557 base. Elective 401(k) deferrals remain subject to Social Security and Medicare tax under IRC §3121(v)(1)(A), so the base is the full $41,943 and the total is $3,208.67, not $3,179.11."
+us,scenario_037,payroll_tax,claude-haiku-4.5,llm_error,other,False,"Correctly applied 6.2% and 1.45% to the full $41,943 and reached about $3,208.74. It then added an unexplained ~$2.90 'minor adjustment' to submit $3,211.64. No adjustment applies: the correct total is $2,600.49 + $608.18 = $3,208.67."
+us,scenario_037,payroll_tax,claude-haiku-5.5,llm_error,missing_output,False,"Named the correct FICA rates but submitted 0 as a placeholder and never computed anything. Applying 7.65% to the $41,943 of wages gives $3,208.67."
+us,scenario_037,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"Subtracted the $5,789 ESI premium from wages, treating it as a pre-tax Section 125 reduction that the household facts never state. FICA applies to the full $41,943, giving $3,208.67, not 7.65% of $36,154."
+us,scenario_037,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Treated the $5,789 employer-sponsored insurance premium as a pre-tax cafeteria-plan reduction, which lowered FICA wages to $36,154. Nothing in the inputs makes these premiums pre-tax, so 6.2% + 1.45% applies to the full $41,943 for $3,208.67."
+us,scenario_037,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"Excluded the $5,789 ESI premium from the Social Security and Medicare base and taxed only $36,154. The premium is not a listed pre-tax salary reduction, so both taxes apply to all $41,943 of wages: $2,600.49 + $608.18 = $3,208.67."
+us,scenario_037,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"Reduced FICA wages by the $5,789 ESI premium, giving $36,154, and computed $2,765.78. The engine taxes gross wages of $41,943 with no premium exclusion, for $3,208.67."
+us,scenario_037,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Assumed the ESI premiums were pre-tax and cut the FICA base to $36,154. The correct base is the full $41,943 of gross wages, and 7.65% of that is $3,208.67."
+us,scenario_037,payroll_tax,gpt-5.4-nano,llm_error,other,False,"Correctly computed about $2,601 + $608 = $3,209 on the full $41,943, then lowered the answer to $3,151 for no stated reason. No state payroll tax or other adjustment applies in NC, so the total is $3,208.67."
+us,scenario_037,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"Correctly kept 401(k) deferrals in FICA wages but subtracted the $5,789 employer-sponsored insurance premium, which leaves a $36,154 base. That premium is not a listed pre-tax salary reduction, so 7.65% applies to the full $41,943, giving $3,208.67."
+us,scenario_037,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"Excluded the $5,789 ESI premium from payroll-tax wages, treating it as a Section 125 pre-tax deduction the inputs never establish. Employee Social Security and Medicare apply to the full $41,943, for $3,208.67."
+us,scenario_037,payroll_tax,grok-4.3,llm_error,other,False,"Used the right base and rates but submitted a rounded $3,210 instead of the exact amount. The precise total is 6.2% × $41,943 ($2,600.49) plus 1.45% × $41,943 ($608.18), which equals $3,208.67."
+us,scenario_037,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"Subtracted the $5,789 ESI premium as pre-tax, giving a $36,154 base, and then rounded each component to whole dollars to reach $2,766. FICA applies to the full $41,943 of wages, giving $3,208.67."
+us,scenario_037,payroll_tax,inkling,llm_error,payroll_tax_base,False,"Treated the $5,789 employer-sponsored health premium as pre-tax and excluded it from FICA wages. The household facts give no pre-tax status, so Social Security and Medicare apply to all $41,943: $2,600.49 + $608.18 = $3,208.67."
+us,scenario_037,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no payroll_tax value and no explanation, so there was nothing to score. The correct derivation is 7.65% of $41,943 in wages, which is $3,208.67."
+us,scenario_037,payroll_tax,minimax-m3,llm_error,other,False,"Correctly calculated $2,601 Social Security plus $608 Medicare on the full $41,943 but added them wrong, reporting $3,213 instead of about $3,209. The exact total is $3,208.67."
us,scenario_037,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an outdated $10,750 standard deduction and a 4.5% rate, getting about $1,386, then set the answer to $0 for no reason. The correct figures are a $12,750 deduction and a 3.99% rate on $28,789.45 of taxable income, and no nonrefundable credit wipes out the tax."
us,scenario_037,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"The model never calculated North Carolina tax and submitted a placeholder $0. Applying 3.99% to $41,539.45 AGI minus the $12,750 standard deduction gives $1,148.70."
us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model first reached about $1,149 correctly. It then dropped the 401(k) reduction, assuming the stated gross wages were already net of 401(k), and taxed $29,193. The stated gross wages are pre-deferral, so NC AGI is $41,539.45."
@@ -3873,52 +3923,53 @@ us,scenario_038,state_refundable_credits,muse-wonder,llm_error,thresholds_rates,
us,scenario_038,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Required Louisiana taxable income or child care activity before any refundable credit applied. The LA EITC is refundable regardless of state liability and equals 5% × $7,316 = $365.80."
us,scenario_038,state_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"Correctly placed the household on the EITC plateau and used the 5% rate, but overstated the 2026 two-child federal maximum as $7,430 instead of $7,316. That gave $371.50 instead of $365.80."
us,scenario_038,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Said Louisiana has no applicable refundable credits, missing the refundable LA EITC of 5% of the $7,316 federal EITC, or $365.80."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model first reached the correct $8,596 under Single filing. It then switched to qualifying-surviving-spouse treatment even though there is no dependent child, applying the joint $32k/$44k Social Security thresholds, the $32,200 joint standard deduction and joint brackets, which produced about $4,488."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"The model treated the childless surviving spouse as a qualifying surviving spouse and used the $32,200 joint standard deduction and the joint 10%/12% brackets. This left taxable income of $47,009 instead of the correct Single taxable income of $63,109, taxed into the 22% bracket."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model correctly found a large positive taxable income, then zeroed the tax with an invented 'elderly/disabled exclusion' and unspecified credits. At this AGI and Social Security level, the elderly/disabled credit is fully phased out, and no other nonrefundable credit applies. The model also made up a disability add-on to the standard deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model claimed the deductions eliminate all liability. In fact, AGI of $79,209 minus the $16,100 Single standard deduction leaves $63,109 taxable, and tax on that is $8,596. Age 65 and disability add-ons do not apply to a 61-year-old who is not blind."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The filing status (Single) was right, but the model miscounted total income as $75,209 instead of $79,209. It also used a $15,750 standard deduction and brackets of $12,150/$49,350 instead of the 2026 values ($16,100; $12,400/$50,400), which understated tax at $7,903."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse status without a dependent child. That brought in the joint $32k/$44k Social Security thresholds (taxable SS of $25,187 instead of $30,689), a roughly $30,000 joint standard deduction and joint brackets, giving $4,749 instead of the Single result of $8,596."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model reached the correct AGI of about $79,209 under Single-type filing but used a $16,700 deduction. Its $6,210 tax on about $62,500 is far below what the Single 10/12/22% brackets give: about $8,460 on that base, and $8,596 on the correct $63,109."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"The model filed the childless surviving spouse under joint brackets with the $32,200 joint standard deduction, which requires a qualifying child. That gave taxable income of $47,009 and tax of $5,145 instead of Single taxable income of $63,109 and tax of $8,596."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model assumed a surviving spouse automatically gets qualifying-surviving-spouse/MFJ treatment, which in fact requires a dependent child. It used the joint SS thresholds (taxable SS of $25,187), a $30,000 standard deduction and a $23,850 joint 10% bracket, producing $4,768."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used qualifying-surviving-spouse treatment for a filer with no dependent child, subtracting a roughly $31,500 joint standard deduction and applying joint brackets. The Single filing status, the $16,100 deduction and the 22% bracket apply."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"The model applied the $32,200 joint standard deduction and joint brackets on the basis of surviving-spouse status. With no qualifying child, the head files Single: taxable income is $63,109 and tax is $8,596."
-us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model correctly used Single status and AGI of $79,209, but subtracted the outdated $15,000 standard deduction and used 2025 bracket thresholds instead of the 2026 OBBBA values ($16,100 deduction; $12,400/$50,400 brackets). That overstated tax at $9,040."
-us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model assumed the TCJA expired and used an $8,800 standard deduction with pre-TCJA 15% rates. OBBBA made the TCJA structure permanent, so 2026 uses the $16,100 Single deduction and 10/12/22% brackets."
-us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"AGI and Single status were right, but the model used a $15,400 standard deduction and brackets of $12,250/$49,800 instead of the 2026 values ($16,100; $12,400/$50,400), which overstated tax at $8,813."
-us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Taxable income of $63,109.25 was correct, but the model used bracket thresholds of $12,200/$49,600 instead of the 2026 Single thresholds of $12,400/$50,400. Too much income fell into the 22% bracket, giving $8,680."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed the TCJA expired in 2026 and applied a combined standard deduction and personal exemption of $13,750 with 15%/25% rates. OBBBA kept the $16,100 Single deduction and the 10/12/22% brackets, with no personal exemption."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. The correct Single computation is $63,109 taxable income and $8,596 tax. The $5,354 answer is consistent with taxing a much smaller base under surviving-spouse/joint 10%/12% brackets."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model had the correct AGI of $79,209 and Single status, but $11,415 is about $2,800 above the correct tax. That is consistent with applying expired pre-TCJA 15%/25% rates instead of the 2026 10/12/22% brackets on $63,109 of taxable income."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed the TCJA sunset and used a $13,400 combined standard deduction and personal exemption with higher pre-TCJA rates. The 2026 Single deduction is $16,100, there is no personal exemption, and the 22% top bracket applies."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no derivation, and its $6,233 understates the tax by about $2,360. The correct computation is AGI $79,209 less the $16,100 Single standard deduction, or $63,109 taxable, which is $8,596 under the 10/12/22% brackets."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model used a $15,700 deduction instead of $16,100. Its $7,117 tax on $63,509 is about $8 below the $7,125 that the joint 10%/12% brackets give on that income and well below the $8,684 under the Single brackets, which tax income above $50,400 at 22%."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"AGI of $79,209 was correct, but $9,414 is about $820 too high. That corresponds to roughly $3,700 too much income in the 22% bracket, which means the model used a smaller standard deduction or lower bracket thresholds than the 2026 values ($16,100 deduction; $50,400 22% threshold)."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model gave no derivation, and its $7,578 understates the tax by about $1,020. Single taxable income of $63,109 produces $8,596 under the 2026 brackets, with $12,709 taxed at 22%."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"AGI of $79,209 was correct, but the model applied a personal exemption and the post-sunset rate schedule. OBBBA made the $16,100 Single standard deduction and the 10/12/22% brackets permanent, with no exemption, so the tax is $8,596 rather than $11,853."
-us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Single status and AGI were right, but the model inflation-projected a $15,432 standard deduction and brackets of $12,259/$49,838 instead of using the 2026 OBBBA values ($16,100; $12,400/$50,400), which overstated tax at $8,802."
-us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model left the $25,950 of estate income and the self-employment loss out of gross income and treated the 61-year-old as a 65-or-older filer. It then subtracted a made-up $1,920 credit for 'previously-taxed Social Security', ending at $2,941."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model claimed taxable income is not positive, but AGI of $79,209 less the $16,100 Single standard deduction leaves $63,109 taxable. There is no elderly or surviving-spouse treatment that offsets this for a 61-year-old with no dependent."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model asserted zero liability from unspecified credits. No nonrefundable credit applies, and the $63,109 of Single taxable income produces $8,596 of tax."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used surviving-spouse joint brackets and the $32,200 standard deduction, which require a dependent child. That gave $47,009 of taxable income instead of the Single $63,109 taxed into the 22% bracket."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model applied surviving-spouse/MFJ treatment with no qualifying child. It used the joint $32k/$44k Social Security thresholds (taxable SS of $25,187 instead of $30,689), the $32,200 deduction and joint brackets, giving $4,485."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model assumed qualifying-surviving-spouse status and used the joint Social Security thresholds and the $32,200 standard deduction, which shrank taxable income to $41,507. The Single filer here has $63,109 of taxable income."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"The model used surviving-spouse joint treatment without a qualifying child. It also gave a 61-year-old the age-65 additional standard deduction and the OBBBA senior deduction, both of which require age 65 or older, leaving only $39,359 of taxable income."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"The model used surviving-spouse filing with the $32,200 joint standard deduction and joint brackets. With no dependent child, the head files Single with the $16,100 deduction, leaving $63,109 taxable and $8,596 of tax."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"The model applied the $32,200 joint standard deduction on the basis of surviving-spouse status, even though that status requires a qualifying child. This understated taxable income at $47,009 instead of $63,109 and missed the 22% bracket."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,household_unit_or_filing_status,False,"The model applied surviving-spouse/MFJ treatment, meaning joint Social Security thresholds, the joint deduction and joint brackets, to a childless widow(er). That gave $4,485; Single filing gives $8,596."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model gave a rounded $8,000 guess and mentioned subtracting medical expenses. Total itemized deductions (about $3,794) are smaller than the $16,100 standard deduction, and the exact Single computation on $63,109 is $8,596."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed the TCJA sunset and used an $8,255 standard deduction, a $5,265 personal exemption and 15%/25% rates. Under OBBBA, 2026 keeps the $16,100 Single deduction and the 10/12/22% brackets with no exemption."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied post-sunset law ($8,300 deduction, $5,300 exemption, 15%/25% rates). OBBBA made the TCJA parameters permanent, so 2026 tax on $63,109 under the 10/12/22% brackets is $8,596."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"The model correctly chose Single, but used a TCJA-sunset $8,500 deduction, a $5,400 exemption and 15%/25% brackets. The 2026 law uses the $16,100 deduction and the 10/12/22% brackets."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"Single status and AGI were right, but the model used a projected $15,420 deduction and brackets of $12,259/$49,832 instead of the 2026 OBBBA values ($16,100; $12,400/$50,400), which overstated tax at $8,805."
-us,scenario_039,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Taxable income of $63,109 was correct, but the model used bracket thresholds of about $12,250/$49,825 instead of the 2026 Single thresholds of $12,400/$50,400. The extra income in the 22% bracket gave $8,656."
-us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,No value or explanation was submitted for federal_income_tax_before_refundable_credits.
-us,scenario_039,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model added the taxable $30,689 of Social Security on top of gross income that already included the full $36,105 of benefits, double-counting them. It also invented extra standard deductions for age 61 and for disability, ending at about $14,000."
-us,scenario_039,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"The $8,798 answer matches taxing the correct $63,109 with the 2025 Single bracket thresholds ($11,925/$48,475) instead of the 2026 thresholds ($12,400/$50,400). This put about $1,925 too much income in the 22% bracket."
-us,scenario_039,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model used qualifying-surviving-spouse/MFJ treatment for a filer with no dependent child. It recomputed taxable Social Security with the joint $32k/$44k thresholds ($25,197), then subtracted the $32,200 joint deduction and applied joint brackets to get $4,505."
-us,scenario_039,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The derivation is internally inconsistent. The model halved the self-employment loss, claimed $90,564 of taxable income after a $31,900 deduction (arithmetically impossible from these inputs), and subtracted a made-up 'credit for other dependents' even though there are no dependents."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"It first reached the correct single-filer tax of $8,596, then switched to surviving-spouse status. It applied the $32,200 joint standard deduction, joint brackets and the $32,000/$44,000 joint Social Security thresholds. That status requires a dependent child, so this childless widow(er) files single: taxable income is $63,109 and tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,household_unit_or_filing_status,False,"It treated the head as a qualifying surviving spouse and used the $32,200 joint standard deduction and joint brackets, giving $5,145.11. With no dependent child, the head files single: $16,100 deduction, $63,109 taxable income, and 22% bracket tax of $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse joint rates and the $32,200 deduction even though there is no dependent child, so single status applies. It also took a $3,938 QBI deduction on estate income, which is not qualified business income. The only business activity is a $6,260 loss, so the QBI deduction is zero and single-filer tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It misstated AGI as $84,395 instead of $79,209 and invented a disability add-on to the standard deduction. It then zeroed the tax with the credit for the elderly or disabled. That credit is fully phased out: the $5,000 base is more than wiped out by $5,415.75 of nontaxable Social Security and half of AGI above $7,500. Tax on $63,109 of single taxable income is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,household_unit_or_filing_status,False,"It claimed age-65 and disability add-ons to the standard deduction that eliminate all tax. The head is 61, and disability does not raise the standard deduction. AGI of $79,209 less the $16,100 single deduction leaves $63,109 taxable, so tax is $8,596.03, not $0."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It chose single status correctly but added the income items to $75,209 instead of $79,209, dropping $4,000. It also used the 2025 $15,750 deduction and guessed brackets of 12,150/49,350 instead of the 2026 $16,100 deduction and 12,400/50,400 brackets. Those errors understate taxable income and give $7,903 instead of $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It filed as a qualifying surviving spouse, using the joint Social Security thresholds (taxable SS of $25,187), a ~$30,000 joint deduction and joint brackets. That status requires a dependent child. As a single filer, SS is taxable at the 85% cap of $30,689, the deduction is $16,100, and tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"It used roughly single treatment with AGI of $79,209 but a $16,700 deduction instead of $16,100. It also misapplied the brackets. The 2026 single schedule on its own ~$62,500 taxable income gives about $8,462, not $6,210, and the correct $63,109 taxable income gives $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse joint brackets and the $32,200 joint deduction to get $5,145.11. With no dependent child the head files single, so the $16,100 deduction leaves $63,109 taxable, which reaches the 22% bracket for $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It assumed qualifying-surviving-spouse status lasts two years after the spouse's death without checking for the required dependent child. It then used joint SS thresholds (taxable SS of $25,187), a $30,000 deduction and 2025 joint brackets. As a single filer, taxable SS is $30,689, taxable income is $63,109, and tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It applied surviving-spouse joint treatment with a ~$31,500 deduction and 2025 joint brackets, then adjusted to $5,187. With no dependent child the head files single: $16,100 deduction, $63,109 taxable income, and $8,596.03 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,household_unit_or_filing_status,False,"It used the surviving-spouse $32,200 joint deduction and joint 10%/12% brackets, giving $5,145. Surviving-spouse status requires a dependent child, so single status applies. The $16,100 deduction leaves $63,109 taxable and $8,596.03 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,household_unit_or_filing_status,False,"It got single status and AGI of $79,209 right but used the pre-OBBBA 2025 $15,000 deduction and 2025 brackets (11,925/48,475). The 2026 values are a $16,100 deduction and brackets ending at $12,400 and $50,400, giving $8,596.03 instead of $9,040."
+us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"It assumed the TCJA expired for 2026 and used an $8,800 deduction with pre-TCJA 10%/15% brackets. It then also ignored the 25% bracket and the personal exemption. OBBBA kept the 10/12/22% schedule and set a $16,100 single deduction, so tax on $63,109 is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,household_unit_or_filing_status,False,"It used single status and the correct AGI but a $15,400 deduction and bracket edges of $12,250/$49,800. The 2026 values are $16,100 and $12,400/$50,400. That overstated taxable income by $700 and pushed more income into 22%, giving $8,813.04 instead of $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,household_unit_or_filing_status,False,"It correctly reached $63,109.25 of taxable income with the $16,100 deduction but used bracket edges of $12,200 and $49,600. The 2026 single edges are $12,400 and $50,400, so it taxed $800 too much at 22%. Correct tax is $8,596.03, not $8,680.04."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"It assumed the TCJA expired in 2026 and applied a $13,750 deduction plus exemption with 10%/15%/25% brackets. OBBBA made the TCJA schedule permanent. With a $16,100 single deduction and the 10%/12%/22% brackets, tax on $63,109 is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"It gave no computation. Its $5,354 is in the range produced by joint-rate surviving-spouse treatment (about $30,000–$32,200 deduction, 10%/12% joint brackets). With no dependent child the head files single, so $63,109 of taxable income reaches the 22% bracket and tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"Its $11,415 on single AGI of $79,209 matches the pre-TCJA 15%/25% rate schedule, not current law. The 2026 10%/12%/22% schedule with a $16,100 deduction gives $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"It applied 'post-TCJA' sunset rules: a ~$13,400 deduction plus personal exemption and the 15%/25% brackets, giving $11,365. OBBBA kept the TCJA rates and set a $16,100 single deduction, so tax on $63,109 is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,household_unit_or_filing_status,False,"It gave no derivation. Under the 2026 single schedule, its $6,233 implies only about $52,400 of taxable income, about $10,700 too low. Correct taxable income is $79,209 AGI less the $16,100 deduction, or $63,109, which gives $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"It had the correct AGI but used a $15,700 deduction instead of $16,100 and then misapplied the brackets. The 10/12/22% single schedule on its own $63,509 taxable income gives about $8,684, not $7,117. The correct $63,109 gives $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"It had the correct $79,209 AGI but overstated tax at $9,414. Under the 2026 single schedule that figure implies a deduction of only about $12,400 instead of the $16,100 OBBBA standard deduction, which leaves $63,109 taxable and $8,596.03 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,household_unit_or_filing_status,False,"It gave no figures. Under the 2026 single schedule, $7,578 implies about $58,500 of taxable income, roughly $4,600 below the correct $63,109 ($79,209 AGI less the $16,100 deduction). Correct tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,household_unit_or_filing_status,False,"It applied a 2026 personal exemption and the expired pre-TCJA rate schedule to AGI of $79,209, giving $11,852.81. OBBBA eliminated the exemption permanently and kept the 10/12/22% brackets with a $16,100 single deduction, so tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"It computed AGI correctly but inflation-projected a $15,432 deduction and bracket edges of $12,259/$49,838. The 2026 values are $16,100 and $12,400/$50,400, so its taxable income was too high and the bracket split was wrong: $8,802 instead of $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,household_unit_or_filing_status,False,"It left the $25,950 of estate income and the $6,260 SE loss out of gross income, so taxable income was understated. It then subtracted a nonexistent $1,920 'tax on previously-taxed Social Security' credit. Correct AGI is $79,209, taxable income is $63,109 after the $16,100 deduction, and tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"It claimed the standard deduction and 'elderly/surviving-spouse treatment' left no taxable income. The head is 61 with no age add-on, and AGI of $79,209 far exceeds the $16,100 single deduction, leaving $63,109 taxable and $8,596.03 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"It asserted that taxable income was low and that credits eliminated the tax, without computing either. AGI is $79,209 including $30,689 of taxable Social Security, taxable income is $63,109, and no nonrefundable credits apply, so tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse brackets and the $32,200 joint deduction even though the head has no dependent child. Single status applies, so the $16,100 deduction leaves $63,109 taxable and tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"It applied surviving-spouse/MFJ treatment, including the $32,000/$44,000 joint SS thresholds (taxable SS of $25,187) and the joint deduction and brackets. With no dependent child the head is single: taxable SS is $30,689, AGI is $79,209, and tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It used qualifying-surviving-spouse joint SS thresholds, the $32,200 deduction and joint brackets, giving $4,485. That status requires a dependent child, so the head files single with $30,689 of taxable SS, $63,109 of taxable income and $8,596.03 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse joint rates without the required dependent child. It also added an age-65 additional deduction and the senior deduction for a 61-year-old, which apply only at 65+. Single status with only the $16,100 deduction gives $63,109 taxable and $8,596.03 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse status with the $32,200 joint deduction and joint brackets, but with no dependent child the head files single. The $16,100 single deduction leaves $63,109 taxable, which reaches the 22% bracket for $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,household_unit_or_filing_status,False,"It applied surviving-spouse joint treatment ($32,200 deduction, joint brackets) to a widow(er) with no qualifying child. The single deduction of $16,100 and single brackets give $8,596.03 on $63,109 of taxable income."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse treatment: joint SS thresholds, the joint deduction and joint brackets, giving $4,484.80. Without a dependent child the head is single, so 85% of SS ($30,689) is taxable, taxable income is $63,109, and tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It cited medical expenses as reducing taxable income alongside the standard deduction. Itemized deductions total only $3,793.91, so the taxpayer takes the $16,100 standard deduction alone and medical costs give no further reduction. Its rounded $8,000 understates the $8,596.03 tax on $63,109 of single taxable income."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"It assumed the TCJA sunset: an $8,255 deduction plus a $5,265 personal exemption and 10%/15%/25% brackets, giving $10,882. OBBBA kept the TCJA brackets and set a $16,100 single deduction with no exemption, so tax on $63,109 is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"It applied TCJA-sunset rules: an $8,300 deduction, a $5,300 exemption and 15%/25% brackets. Under OBBBA the 2026 single deduction is $16,100 with the 10%/12%/22% brackets, giving $8,596.03 rather than $10,742."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,household_unit_or_filing_status,False,"It chose single status correctly but assumed the TCJA had sunset, using an $8,500 deduction, a $5,400 exemption and 15%/25% brackets. OBBBA's permanent extension sets a $16,100 deduction and the 10/12/22% schedule, giving $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"It used single status and the right AGI but projected a $15,420 deduction and bracket edges of $12,259/$49,832. The 2026 values are $16,100 and $12,400/$50,400, which give $63,109 taxable and $8,596.03 instead of $8,805."
+us,scenario_039,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"It correctly reached $63,109 of taxable income with the $16,100 deduction but used bracket edges of about $12,250 and $49,825. The 2026 single edges are $12,400 and $50,400, so it taxed too much at 22% and got $8,656 instead of $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for this output, so there is no answer to score against the $8,596.03 reference."
+us,scenario_039,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"It counted Social Security twice, adding the full $36,105 to gross income and then the $30,689 taxable portion on top. It also invented age-61 and disability add-ons to the standard deduction. AGI is $79,209 and taxable income is $63,109, so tax is $8,596.03, not $14,000."
+us,scenario_039,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,household_unit_or_filing_status,False,"It used single status and the 10/12/22% brackets on AGI of $79,209 but not the exact 2026 parameters. With the $16,100 deduction and bracket edges of $12,400/$50,400, tax is $8,596.03, not $8,798."
+us,scenario_039,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"It treated the head as a qualifying surviving spouse, applying the $32,000/$44,000 joint SS thresholds, the $32,200 deduction and 2025 joint brackets, even though there is no dependent child. As a single filer, $30,689 of SS is taxable, taxable income is $63,109, and tax is $8,596.03."
+us,scenario_039,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It used a $31,900 joint-level deduction and halved the SE loss. It then subtracted a $1,716 credit for other dependents tied to the pension, which requires a dependent the household does not have. With no credits and single status, tax on $63,109 of taxable income is $8,596.03."
us,scenario_039,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_039,head_medicaid_eligible,claude-sonnet-5,llm_error,health_coverage,False,"Virginia's expansion adult group covers 19-64-year-olds only up to 138% FPL, yet the model declared that countable MAGI ""falls within eligible range for a household size of 1"" without ever summing the listed income: $36,105 of Social Security counts in full under MAGI rules, on top of $26,800 in IRA distributions, $2,030 in pension, and $25,950 in estate income, against which the $6,260 self-employment loss is trivial, putting the head at 368% FPL. Its correct observation that the expansion group applies no asset test is irrelevant once income blows through the 138% FPL ceiling, and the SSI-related aged/blind/disabled group it gestured at requires income near the SSI standard with SSI receipt at $0 here."
us,scenario_039,head_medicaid_eligible,glm-5.3,llm_error,health_coverage,False,"The model named the correct rule — Virginia expanded to 138% FPL — then asserted the head's MAGI was below that threshold without computing it. Medicaid MAGI here is 368% FPL: full Social Security of $36,105, $26,800 in IRA distributions, $2,030 in pension, and $25,950 in estate income, less the $6,260 self-employment loss. With income at 2.7x the expansion limit and no SSI receipt to open the aged/blind/disabled route, the head qualifies through no Virginia category."
@@ -4030,57 +4081,58 @@ us,scenario_040,state_income_tax_before_refundable_credits,minimax-m3,llm_error,
us,scenario_040,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model taxed $21,420 of Social Security as part of the Arizona base, missing both that provisional income of $40,600 puts only $4,300 of the $25,200 in federal AGI and that Arizona subtracts 100% of the Social Security benefits included in federal AGI. It also contradicted its own arithmetic, computing $484.25 and then submitting $486.25; with Social Security removed, Arizona AGI is $28,000, below the roughly $32,200 MFJ standard deduction, so taxable income and tax are both zero."
us,scenario_040,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model conflated Arizona’s distinct excise-tax and property-tax credits and treated age plus an asserted $25,000 AGI ceiling as sufficient for a $200 refundable credit. It never applied the specific eligibility requirements for either Arizona credit; this household qualifies for neither, so az_refundable_credits is $0."
us,scenario_040,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted state_refundable_credits entirely instead of submitting the required numeric value and explanation. The required output is $0 because every state and locality refundable-credit component, including Arizona’s, evaluates to zero."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model left the $928 of qualified dividends out of non-Social-Security income ($32,807 instead of $33,735). That understated provisional income, so it got $16,057 of taxable Social Security and AGI of $48,864 instead of $16,846 and $50,581. The resulting taxable income of $24,714 is $1,717 below the correct $26,431."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The model filed as a qualifying surviving spouse and used the $32,200 joint standard deduction plus the $1,650 aged add-on. QSS status requires a dependent child, and this head has none, so they file single with $16,100 + $2,050 + $6,000. Its taxable income of about $10,717 is roughly $15,700 too low."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model counted the full $29,580 of Social Security as income instead of the $16,846 taxable portion, and it skipped the $6,000 senior deduction. It also taxed the $4,681 of qualified dividends and gains at 15% instead of 0%, and it subtracted an invented $783 'dependent exemption credit'."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model said taxable income is close to zero, but AGI of $50,581 minus $24,150 of deductions leaves $26,431 taxable. Of that, $21,750 is ordinary income taxed at 10% and 12%. It treated the deductions and the Social Security exclusion as wiping out the whole tax base."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model treated the $29,580 of Social Security dependent benefits as belonging to someone else and left them out completely. These are the head's benefits, and $16,846 of them is taxable. It also dropped the $928 of qualified dividends and left out the $6,000 senior deduction, so its taxable income of $14,657 is far too low."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used qualifying surviving spouse status with a deduction of about $32,350, but the head has no dependent child and must file single. It also took taxable Social Security as the 85% cap ($25,143) instead of the formula amount of $16,846. Its final $1,245 was an unexplained adjustment rather than a computed result."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model set taxable Social Security at the 85% cap of $25,143 instead of using the formula 0.85×(48,525−34,000)+4,500 = $16,846. That overstated AGI by about $8,300, at $58,878. Its tax figure also doesn't follow from its own stated taxable income."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model filed as a qualifying surviving spouse and used the $32,200 + $1,650 joint standard deduction. With no dependent child, the head files single, with a $16,100 standard deduction and a $2,050 aged add-on. That error cut taxable income to $10,717 instead of $26,431."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model left out the $6,000 OBBBA senior deduction for filers 65 and older. It also used outdated standard deduction figures ($15,750 + $2,000 instead of $16,100 + $2,050) and a 10% bracket ending at $12,200 instead of $12,400. Its taxable income came to $32,949, against the correct $26,431."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model left out the head's Social Security entirely by treating it as a dependent's benefit, when $16,846 of it is taxable. Then, even after reaching about $16,792 of taxable income by its own figures, it reported $0 tax. Its ordinary income above the 0%-rate items should have been taxed at 10% and 12%."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model used surviving-spouse status with the $32,200 joint standard deduction and the $1,650 aged add-on. A widowed filer with no dependent child files single, with $16,100 + $2,050 + $6,000 of deductions. That error left only $6,050 of ordinary income taxed at 10%."
-us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model left out the $6,000 senior deduction and used a $17,650 standard deduction instead of $18,150. That gave taxable income of $33,063 instead of $26,431. It also added the $71 of BDC income as a separate income item."
-us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied only a $17,425 standard deduction and left out the $6,000 OBBBA senior deduction, so taxable income came to $33,288. The correct deductions are $16,100 + $2,050 + $6,000 = $24,150."
-us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"The model assumed the TCJA expired in 2026 and used an $8,600 standard deduction, a $5,500 personal exemption and the 15% bracket. The OBBBA made the TCJA structure permanent, so 2026 has a $16,100 standard deduction, a $2,050 aged add-on, the $6,000 senior deduction, no personal exemption, and 10%/12% brackets."
-us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"The model used a $17,350 standard deduction and no $6,000 senior deduction. That produced taxable income of $33,363 instead of $26,431, which overstates the 12%-bracket tax."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model assumed the TCJA expired, using a $10,300 standard deduction, a $5,200 personal exemption and a 15% bracket. 2026 law keeps the $16,100 standard deduction, the $2,050 aged add-on and the $6,000 senior deduction, with no personal exemption and 10%/12% brackets."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a head-of-household-over-65 standard deduction and no $6,000 senior deduction. Its $3,156 is close to the $3,082 that results from leaving out the senior deduction. The head files single and gets $24,150 of total deductions."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model started from the correct AGI of about $50,581 but subtracted only the elderly single standard deduction and missed the $6,000 senior deduction. Its round $3,500 overstates even the roughly $3,082 that the no-senior-deduction path gives."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied 'pre-TCJA' 2026 rules, with $15,150 of standard deduction plus personal exemptions and the old 15% bracket. The OBBBA made the TCJA brackets and the larger standard deduction permanent and added the $6,000 senior deduction, which brings taxable income down to $26,431."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. The correct computation is AGI of $50,581 minus $24,150 of deductions, leaving $21,750 of ordinary income taxed at 10%/12%. Its $2,878 is about $516 higher, which matches roughly $4,300 too much taxable income at 12%, as from understating the senior or aged deductions."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model filed as a surviving spouse and took a $31,550 joint-width standard deduction, though the head has no dependent child and files single. It also put taxable Social Security at $9,846 instead of $16,846, so AGI came to $43,581."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted only the single 65+ standard deduction from AGI of about $50,581 and left out the $6,000 senior deduction. Its explanation gives $3,584 while the submitted value is $3,090, and both are above the correct $2,361.96."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"The model applied only the standard deduction and missed the $6,000 senior deduction. Its $3,734 is even higher than the roughly $3,082 that leaving out the senior deduction alone produces, so its income or bracket figures are also too high."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"The model included a personal exemption, which is $0 under permanent TCJA/OBBBA law. That dropped its taxable income below the correct $26,431, which comes from the $24,150 of standard, aged and senior deductions alone."
-us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model left out the $6,000 senior deduction and used a $17,850 standard deduction. It also mis-added non-Social-Security income as $34,806 instead of about $33,806, which inflated taxable Social Security to $17,757. Its taxable income came out at $34,713 instead of $26,431."
-us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"The model treated Social Security as 'largely nontaxable', but $16,846 of the $29,580 is taxable at this income level. Its placeholder-style $1,234.56 is far below the $2,361.96 that follows from $21,750 of ordinary income taxed at 10%/12%."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model claimed the standard and age-related deductions fully offset the income. AGI is $50,581 against $24,150 of deductions, which leaves $26,431 taxable and $21,750 of ordinary income taxed at 10% and 12%."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed deductions and nonrefundable credits reduce the tax to zero. The $24,150 of deductions leave $26,431 of taxable income, and no nonrefundable credit applies; the elderly credit is fully phased out."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used a $33,200 surviving-spouse standard deduction, but QSS requires a dependent child, so the head files single with $16,100 + $2,050. It also didn't add the $6,000 senior deduction, which still left it far too little ordinary income at 10%."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model put AGI at $58,949 by taxing Social Security at the 85% cap ($25,143) instead of the formula amount of $16,846. It then taxed the qualified dividends and capital gains at 15%, although they fall in the 0% band below $49,450."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used qualifying surviving spouse status and its joint-width standard deduction, which left only $6,036 of ordinary income. With no dependent child, the head files single, and ordinary taxable income is $21,750."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used the surviving-spouse standard deduction, which put all ordinary income in the 10% bracket. With no qualifying child, single status and a $16,100 + $2,050 + $6,000 deduction apply, which leaves $21,750 of ordinary income running into the 12% bracket."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"The model used qualifying surviving spouse status and a $33,850 joint-width age-adjusted standard deduction. The head has no dependent child and files single with $18,150 + $6,000, so ordinary taxable income is $21,750, not $6,036."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model's taxable income of $34,785 matches using the 85% cap ($25,143) for taxable Social Security instead of the formula amount of $16,846. It also gave 0% treatment only to the $928 of qualified dividends and taxed the $3,753 of capital gains as ordinary income."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"The model's $2,377.79 is $15.83 above the correct figure, which is about $132 of extra ordinary income at 12%. That matches adding the $71 of qualified BDC income as separate income, which also pushes taxable Social Security up by 0.85×$71."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"The model used surviving-spouse deductions and then claimed only preferential income was left. The head files single, and $24,150 of deductions against $50,581 of AGI leaves $21,750 of ordinary income taxed at 10%/12%."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"The model applied surviving-spouse standard and senior deductions and concluded that taxable income fits inside the 0% capital-gains band. As a single filer, the head has $26,431 of taxable income, of which $21,750 is ordinary income taxed at 10%/12%."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied only the standard deduction, with a minor mortgage-interest and QBI adjustment, and left out the $6,000 senior deduction. Its $3,125 matches taxable income of about $32,000+ instead of $26,431."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used pre-TCJA sunset rules: a $10,507 standard deduction, a $5,387 personal exemption and a 15% bracket. 2026 law has a $16,100 standard deduction, a $2,050 aged add-on and a $6,000 senior deduction, with no exemption and 10%/12% brackets."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model assumed post-TCJA-expiration parameters: an $8,300 standard deduction, a $5,300 exemption and a 15% bracket. The OBBBA made the TCJA rates and the larger standard deduction permanent and added the $6,000 senior deduction."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model used TCJA-sunset parameters: an $8,300 standard deduction, a $2,000 aged add-on, a $5,300 personal exemption and a 15% bracket. 2026 law gives $24,150 of total deductions with 10%/12% brackets, so taxable income is $26,431, not $35,113."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model applied pre-TCJA rules: a $10,233 standard deduction and a 15% bracket. It also overstated taxable Social Security at $24,205 instead of $16,846. On top of that, it gave 0% treatment only to $999 of qualified income and taxed the $3,753 of capital gains as ordinary income."
-us,scenario_042,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model counted the farm loss twice ($2,254 instead of $1,127), which distorted AGI and taxable Social Security. It also deducted only the $18,150 standard deduction and left out the $6,000 senior deduction, so taxable income came to $30,478 instead of $26,431."
-us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for this output. The correct value is $2,361.96, from $21,750 of ordinary taxable income at 10%/12% after $24,150 of deductions."
-us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model's filing status and deductions were correct, but it computed taxable Social Security as $24,108.75. The formula gives 0.85×(48,525−34,000)+4,500 = $16,846.25, so AGI was overstated by $7,262.50 and taxable income came to $33,694 instead of $26,431."
-us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used surviving-spouse/MFJ treatment, which brought in the $32,000/$44,000 Social Security thresholds and a $32,150 standard deduction. The head has no dependent child and files single, so the $25,000/$34,000 thresholds give $16,846 of taxable Social Security, and the deductions are $24,150."
-us,scenario_042,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at about $18,407 instead of $16,846, which gave AGI of $52,213. It also understated deductions at $23,750 instead of $24,150, so taxable income came to $28,463 instead of $26,431."
-us,scenario_042,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"The model applied only the age-65 standard deduction and left out the $6,000 OBBBA senior deduction. Its $3,081 is close to the correct computation with that $6,000 added back to taxable income."
-us,scenario_042,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used surviving-spouse/MFJ treatment, with the $44,000 threshold that cut taxable Social Security to about $9,907 and a $32,200 base standard deduction. The head has no dependent child and files single, which yields $16,846 of taxable Social Security and an $18,150 standard deduction."
-us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model filed as a qualifying surviving spouse and put the $4,700 of tax-exempt IRA distributions into provisional income. It got $25,698 of taxable Social Security, which exceeds the 85% cap. It also left out the $6,000 senior deduction. The head is a single filer with $16,846 of taxable Social Security."
-us,scenario_042,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model gave no derivation. Its $21,824 is close to 9 times the correct tax, which comes from $26,431 of taxable income ($21,750 ordinary at 10%/12% and $4,681 at 0%). The answer matches taxing nearly all gross income at high marginal rates and ignoring the $24,150 of deductions and the 0% rate on qualified income."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used the correct single filing status and the $24,150 of deductions. But it treated the $928 of qualified dividends as part of the $5,920 non-qualified dividends and dropped them from income. That understated non-SS income ($32,807 vs $33,735), taxable SS ($16,057 vs $16,846) and AGI ($48,864 vs $50,581), and its approximate brackets gave $2,133 instead of $2,361.96."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It applied qualifying-surviving-spouse joint parameters ($32,200 standard deduction plus a $1,650 age addition). QSS requires a dependent child, so the head files Single, with a $16,100 deduction plus a $2,050 age addition. It also taxed the $3,753 of capital gains as ordinary income, which left just $10,717 of taxable income instead of $26,430.71."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It used QSS joint standard deductions even though the head has no dependent child and must file Single. It also counted the farm loss twice (-$2,254 instead of -$1,127) and left the $3,753 of capital gains out of gross income. Together these cut taxable income to $1,689 instead of $26,430.71."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It included the full $29,580 of Social Security instead of the $16,846 taxable portion and left out the $6,000 senior deduction. It taxed the $4,681 of qualified dividends and gains at 15% instead of 0%, and it subtracted a made-up $783 'dependent exemption credit'."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed taxable income was close to zero. In fact AGI of $50,580.71 (including $16,846 of taxable SS) minus $24,150 of deductions leaves $26,430.71, and $21,749.71 of that is ordinary income taxed at 10%/12%."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated the $29,580 of Social Security benefits as belonging to someone else and excluded them, which dropped $16,846 of taxable SS. It also left out the $6,000 senior deduction and dropped the $928 of qualified dividends from income, ending at about $14,657 of taxable income instead of $26,430.71."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It used QSS joint brackets and the joint standard deduction, although with no dependent child the head files Single. It also set taxable SS at the 85% cap ($25,143) instead of using the formula result of $16,846, then replaced its own result with an arbitrary $1,245."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It assumed 85% of benefits ($25,143) were taxable. The single-filer formula gives 0.85 × ($48,525 − $34,000) + $4,500 = $16,846. That inflated AGI to about $58,878, and it then adjusted to an unsupported $2,699."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Its arithmetic was right, but it used QSS joint parameters ($32,200 standard deduction plus $1,650). A surviving spouse with no dependent child files Single, with a $16,100 deduction plus $2,050, so its taxable income of $10,717 is understated by $15,714."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It filed Single and correctly computed about $16,907 of taxable SS, but it left out the $6,000 OBBBA senior deduction for filers 65 and older. It also used a standard deduction of about $17,750 instead of $18,150, which left about $6,500 too much income in the 12% bracket."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the $29,580 of Social Security as not taxable to the head, which dropped $16,846 of taxable SS. It then reported $0 even though its own taxable income of about $16,792 included ordinary income above the preferential amounts."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It filed as a qualifying surviving spouse with the $32,200 joint standard deduction. With no dependent child the head files Single ($16,100 + $2,050 + $6,000), so its taxable income of $10,731 is far below $26,430.71."
+us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It filed Single and computed taxable SS correctly, but it left out the $6,000 senior deduction and used a $17,650 standard deduction instead of $18,150. That overstated taxable income by about $6,600 and taxed the extra at 12%."
+us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It left out the $6,000 OBBBA senior deduction and used a $17,425 standard deduction instead of $16,100 + $2,050 = $18,150. Taxable income came out at $33,288 instead of $26,431."
+us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA sunset law: a small standard deduction, a $5,500 personal exemption and a 15% bracket. OBBBA made the TCJA rates and the larger standard deduction permanent and added the $6,000 senior deduction, so 2026 tax uses the $18,150 + $6,000 deductions and the 10%/12% brackets."
+us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It applied only a $17,350 standard deduction and left out the $6,000 OBBBA senior deduction. The correct single aged deduction is $16,100 + $2,050 = $18,150, so its taxable income of $33,363 is overstated by about $6,900, all taxed at 12%."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed TCJA expired and used a $10,300 standard deduction, a $5,200 personal exemption and a 15% bracket. 2026 law under OBBBA keeps the $16,100 + $2,050 standard deduction, adds the $6,000 senior deduction and keeps the 12% bracket."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It applied only a standard deduction, and it wrongly called the head a head-of-household filer. It left out the $6,000 senior deduction, and its $3,156 matches taxing about $6,000 too much income at 12%."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It reached the correct AGI of about $50,581 but subtracted only the elderly-single standard deduction and left out the $6,000 senior deduction. It then rounded further up to $3,500, while the bracket math on $26,431 of taxable income gives $2,362."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 law with personal exemptions and the 15% bracket, using only $15,150 of combined deductions. OBBBA kept the TCJA brackets and the $18,150 aged standard deduction and added the $6,000 senior deduction."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path is AGI $50,580.71 minus $24,150 = $26,430.71, with $21,749.71 ordinary at 10%/12% giving $2,361.96. Its $2,878 corresponds to about $4,300 of extra ordinary taxable income, consistent with omitting most of the $6,000 senior deduction."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used a QSS standard deduction of $31,550 even though the head has no dependent child and files Single. It also understated taxable SS at $9,846 instead of $16,846 and left out the $6,000 senior deduction."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It reached AGI of about $50,581 but subtracted only the elderly-single standard deduction and left out the $6,000 senior deduction. Its submitted $3,090 matches the roughly $3,082 that results from dropping that deduction, and its explanation contradicts itself by stating $3,584."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It subtracted only the standard deduction and left out the $6,000 OBBBA senior deduction. Its $3,734 also exceeds the roughly $3,082 that omission alone would produce, so taxable income was overstated further."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It applied a personal exemption, which was eliminated permanently by TCJA/OBBBA, and it did not apply the $16,100 + $2,050 + $6,000 deduction structure. Its $2,235 does not match the correct $26,430.71 taxable income taxed at 10% up to $12,400 and 12% above."
+us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added the non-SS items wrong, reporting $34,806 when they total $33,806, which also inflated taxable SS to $17,757. It also left out the $6,000 senior deduction for filers 65 and older, so taxable income came out at $34,713 instead of $26,431."
+us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It gave no computation and submitted the placeholder $1,234.56. The correct path taxes $21,749.71 of ordinary income ($26,430.71 taxable income minus $4,681 of preferential income) at 10%/12%, which gives $2,361.96."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard and age deductions fully offset income. In fact AGI of $50,580.71, including $16,846 of taxable SS, exceeds the $24,150 of deductions by $26,430.71, leaving $21,749.71 of ordinary income taxed at 10%/12%."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that deductions and credits eliminate the tax. No nonrefundable credit applies, and $26,430.71 of taxable income remains after the $24,150 of deductions, producing $2,361.96 of tax."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used a surviving-spouse standard deduction of about $33,200, but a widow with no dependent child files Single with $16,100 + $2,050. It also left out the $6,000 senior deduction, so its taxable income of about $17,400 is wrong in both directions."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It treated 85% of benefits as taxable, giving AGI of $58,949; the single-filer formula gives $16,846 of taxable SS and AGI of $50,581. It also taxed the $4,681 of qualified dividends and gains at 15%, although they fall entirely within the 0% band."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the QSS joint standard deduction, which requires a dependent child. The head has none and files Single ($16,100 + $2,050 + $6,000), so its ordinary taxable income of $6,036 is understated by $15,714."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It used the surviving-spouse joint standard deduction and placed all ordinary income in the 10% bracket. With no dependent child the head files Single, which leaves $21,749.71 of ordinary income extending well into the 12% bracket."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used qualifying-surviving-spouse status with the $33,850 joint age-adjusted standard deduction. QSS requires a dependent child, so the head files Single with $18,150, and ordinary taxable income is $21,749.71, not $6,036.05."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its taxable income of $34,785 implies it treated 85% of benefits ($25,143) as taxable instead of $16,846. It also gave 0% treatment only to the $928 of qualified dividends and taxed the $3,753 of capital gains as ordinary income."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It used the correct single structure but overshot by about $16, which equals 12% of about $132 of extra taxable income. That matches adding the $71 qualified BDC income as separate ordinary income, which also pulls in another $60 of taxable SS. PolicyEngine does not include that $71 in gross income."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used QSS status although the head has no dependent child and files Single. It also claimed only preferential income remained, but even its own QSS deductions leave about $6,036 of ordinary income taxed at 10%."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It applied the surviving-spouse joint standard deduction, although the head files Single with no dependent child. It concluded that all taxable income fell in the 0% capital-gains band, when $21,749.71 of ordinary income remains after $24,150 of single deductions."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It applied only the standard deduction, along with a nonexistent mortgage-interest deduction, and left out the $6,000 OBBBA senior deduction. Its $3,125 corresponds to taxing about $6,000 too much income at 12%."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It assumed TCJA sunset in 2026 and used a $10,507 standard deduction, a $5,387 personal exemption and a 15% bracket. OBBBA made the TCJA rates permanent, so the correct calculation uses $18,150 + $6,000 of deductions and the 10%/12% brackets."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used post-TCJA-expiration parameters: an $8,300 standard deduction, a $5,300 exemption and the 15% bracket. 2026 law under OBBBA keeps the $16,100 standard deduction, the $2,050 aged addition and the 12% bracket, and adds the $6,000 senior deduction."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA law: an $8,300 standard deduction, a $2,000 aged addition, a $5,300 exemption and a 15% bracket. OBBBA made the TCJA brackets and the larger standard deduction permanent and added the $6,000 senior deduction, giving taxable income of $26,430.71 instead of $35,113."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used reverted pre-TCJA brackets with a 15% rate and a $10,233 deduction, and it overstated taxable SS at $24,205 instead of $16,846. It also gave 0% treatment only to $999 of qualified income and taxed the $3,753 of capital gains as ordinary income."
+us,scenario_042,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It counted the farm loss twice (-$2,254) and left out the $6,000 senior deduction, subtracting only the $18,150 standard deduction. Taxable income came out at $30,478 instead of $26,430.71."
+us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no parseable value or explanation for federal_income_tax_before_refundable_credits.
+us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly filed Single and applied $24,150 of deductions, but it miscomputed taxable SS as $24,108.75. The single formula gives 0.85 × ($48,525 − $34,000) + $4,500 = $16,846.25, so AGI was overstated by $7,262.50."
+us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated the head as a surviving spouse with joint parameters: the MFJ $32k/$44k SS thresholds, giving $8,407 of taxable SS, and a $32,150 standard deduction. With no dependent child the head files Single, using the $25k/$34k thresholds ($16,846 taxable SS) and the $24,150 single deductions."
+us,scenario_042,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It used the correct single structure but overstated taxable SS at $18,407 instead of $16,846. It also understated total deductions at $23,750 instead of $16,100 + $2,050 + $6,000 = $24,150, so about $2,000 of extra income was taxed at 12%."
+us,scenario_042,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It applied only the age-65 standard deduction and left out the $6,000 OBBBA senior deduction. That adds $720 at 12% to the $2,362 result, matching its $3,081."
+us,scenario_042,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used MFJ rates, MFJ SS thresholds ($9,907 of taxable SS) and the $32,200 joint standard deduction. A surviving spouse with no dependent child files Single. It also taxed the $3,753 of capital gains as ordinary income."
+us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It filed as a qualifying surviving spouse even though there is no dependent child. It added the tax-exempt IRA distributions to provisional income and set taxable SS at $25,698, which exceeds the 85% cap. It also left out the $6,000 senior deduction."
+us,scenario_042,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $21,824 is about the size of the $21,749.71 ordinary taxable income, not the tax on that income. Applying the 10% rate up to $12,400 and 12% above to $21,749.71 gives $2,361.96."
us,scenario_042,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_042,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no employee wages were listed and that payroll tax should therefore be zero, but submitted $1,530 anyway. That amount contradicts its own payroll-tax-base reasoning and has no listed wage base supporting it."
us,scenario_042,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model provided no payroll-tax output. It failed the required structured-output contract rather than completing the computation from the zero listed wage base.
@@ -4139,85 +4191,88 @@ us,scenario_043,head_medicaid_eligible,qwen3.8-max,llm_error,age_disability,Fals
us,scenario_043,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_043,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_043,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_043,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"The model correctly included the Colorado FAMLI employee premium but used the 2023-2025 rate of 0.45% instead of the 2026 rate of 0.44% (half of 0.88%), getting $331.14. It then submitted $315.14, which matches none of its own figures, so the number is a transcription error on top of the rate error."
-us,scenario_043,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"The model computed FICA correctly at $312.73 but left out Colorado's FAMLI employee premium (0.44% × $4,088.15 = $17.99), which the reference counts as a state payroll tax. It read 'mandatory employee state payroll taxes' as excluding FAMLI because the employer is allowed to pay that share."
-us,scenario_043,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no mandatory employee payroll tax, so it left out the $17.99 FAMLI employee premium (0.44% of FAMLI wages). It submitted only FICA, rounded to $313."
-us,scenario_043,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"The model counted only FICA ($312.74, rounded to $313) and never applied Colorado's FAMLI employee premium of 0.44% on $4,088.15, or $17.99, which the reference includes."
-us,scenario_043,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"The model identified the Colorado FAMLI employee premium but dropped it, treating it as 'state UI-like.' The reference counts it as an employee state payroll tax at 0.44% (not the 0.45% the model cited), adding $17.99."
-us,scenario_043,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no mandatory employee payroll tax and skipped the FAMLI employee premium (0.44% × $4,088.15 = $17.99). That leaves FICA alone, $312.73, instead of $330.73."
-us,scenario_043,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"The model counted only federal FICA (about $313) and left out the Colorado FAMLI employee contribution of 0.44% of FAMLI wages ($17.99), which the reference includes as a state payroll tax."
-us,scenario_043,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,The model applied only the 7.65% FICA rate ($312.73) and never added Colorado's 0.44% FAMLI employee premium ($17.99) to the payroll tax total.
-us,scenario_043,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no mandatory employee-side payroll tax, which misses the FAMLI program's 0.44% employee premium on $4,088.15 of wages ($17.99). It reported FICA alone."
-us,scenario_043,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"The model looked only for state disability or unemployment-insurance contributions and missed Colorado's FAMLI paid-leave premium. The employee share, 0.44% × $4,088.15 = $17.99, raises the total from $312.73 to $330.73."
-us,scenario_043,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,The model said Colorado has no employee state payroll tax and left out the $17.99 FAMLI employee premium (0.44% of FAMLI wages). It reported only the $312.73 of FICA.
-us,scenario_043,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"The model correctly included the Colorado FAMLI employee premium but put it at $24.53, which is about 0.6% of wages. The 2026 employee rate is 0.44% (half of the 0.88% premium), which gives $17.99 on $4,088.15, for a total of $330.73."
-us,scenario_043,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"The model treated the $8,389 employer-sponsored insurance premium as a pre-tax deduction from the employee's wages and set the FICA and FAMLI wage base to $0. The prompt says nothing about a Section 125 salary reduction, so the full $4,088 of wages is taxable: $253.47 Social Security, $59.28 Medicare, and $17.99 FAMLI."
-us,scenario_043,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"The model applied only the 7.65% FICA rate to $4,088 and left out Colorado's FAMLI employee premium of 0.44% ($17.99), which the reference counts as a state payroll tax."
-us,scenario_043,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"The model's FICA total of $312.74 is correct, but it left out the Colorado FAMLI employee contribution (0.44% × $4,088.15 = $17.99) that brings the total to $330.73."
-us,scenario_043,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,The model reported only Social Security and Medicare ($312.73) and skipped the 0.44% Colorado FAMLI employee premium ($17.99) included as a state payroll tax.
-us,scenario_043,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"The model computed FICA of $312.73 and stopped there, leaving out Colorado's FAMLI employee premium of 0.44% on $4,088.15 ($17.99)."
-us,scenario_043,payroll_tax,gemini-4-argon,prompt_ambiguity,state_local_rule,False,The model applied only the 7.65% FICA rate and left the Colorado FAMLI employee premium ($17.99 at 0.44%) out of the payroll tax total.
-us,scenario_043,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"The model computed FICA of $312.73, rounded it to $313, and left out the 0.44% Colorado FAMLI employee premium ($17.99) that the reference adds."
-us,scenario_043,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no state employee payroll tax, which leaves out the $17.99 FAMLI employee premium (0.44% of FAMLI wages). It also truncated FICA of $312.74 down to $312."
-us,scenario_043,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"The model gave no real basis for zeroing out payroll tax. The $4,088 of wages is fully subject to 6.2% Social Security ($253.47), 1.45% Medicare ($59.28), and the 0.44% Colorado FAMLI employee premium ($17.99)."
-us,scenario_043,payroll_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"The model deliberately left out any Colorado employee payroll tax and reported FICA alone ($312.73). The reference adds the FAMLI employee premium at 0.44% of $4,088.15, or $17.99."
-us,scenario_043,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"The model computed only the 7.65% FICA on $4,088 ($312.73) and never applied Colorado's 0.44% FAMLI employee contribution ($17.99)."
-us,scenario_043,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"The model counted only Social Security and Medicare ($312.73, rounded to $313) and left out the Colorado FAMLI employee premium of $17.99."
-us,scenario_043,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"The model's FICA base and rates are correct, but it said Colorado has no employee payroll tax. That misses the FAMLI employee premium of 0.44% × $4,088.15 = $17.99."
-us,scenario_043,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,The model correctly kept 401(k) deferrals in the FICA base but said Colorado has no employee payroll tax. That leaves out the 0.44% FAMLI employee premium ($17.99) the reference includes.
-us,scenario_043,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"The model said no mandatory state payroll taxes apply, which misses Colorado's FAMLI employee premium (0.44% of FAMLI wages, $17.99). It submitted FICA alone, rounded to $313."
-us,scenario_043,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"The model computed FICA of $312.73 and left out the Colorado FAMLI employee premium of 0.44% on $4,088.15 ($17.99) that brings the total to $330.73."
-us,scenario_043,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no mandatory employee state payroll tax and left out the $17.99 FAMLI employee premium (0.44% of FAMLI wages). It reported FICA alone, rounded to $313."
-us,scenario_043,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,The model counted only Social Security and Medicare ($312.73) and never applied Colorado's 0.44% FAMLI employee premium ($17.99).
-us,scenario_043,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"The model said Colorado has no mandatory employee state payroll tax, which leaves out the FAMLI employee premium (0.44% × $4,088.15 = $17.99). It reported FICA alone."
-us,scenario_043,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,The model looked only for a state payroll 'income' tax and missed Colorado's FAMLI paid-leave premium. The employee share at 0.44% adds $17.99 to the $312.73 of FICA.
+us,scenario_043,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It correctly included a Colorado FAMLI employee premium but used 0.45% ($18.40) instead of the 2026 rate of 0.44% ($17.99), which gave $331.14 rather than $330.73. It then submitted $315.14, a number that matches neither its own FICA-plus-FAMLI total nor its FICA-only total of $312.73. The final answer is a transcription error."
+us,scenario_043,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It assumed no Colorado employee payroll tax applies and counted only FICA ($312.73). That leaves out the 0.44% FAMLI employee premium ($17.99) that PolicyEngine withholds from wages. Because the prompt never says whether the employer pays the employee's half, both readings are possible."
+us,scenario_043,payroll_tax,claude-fable-5.2,prompt_ambiguity,state_local_rule,False,"It explicitly assumed the FAMLI employee premium was not included and counted only FICA ($312.73). The reference instead withholds the 0.44% employee share ($17.99) from the $4,088 in wages. The prompt doesn't say whether the employer covers that share."
+us,scenario_043,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It stated that Colorado has no mandatory state payroll tax and gave FICA only ($312.74, rounded to $313). That leaves out the 0.44% FAMLI employee premium ($17.99) that the reference withholds. Colorado lets the employer pay that share, and the prompt doesn't say who pays."
+us,scenario_043,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"It counted only employee Social Security and Medicare ($312.74, rounded to $313), leaving out the 0.44% Colorado FAMLI employee premium ($17.99) in the reference. It matches the reading in which the employer covers the employee's FAMLI share."
+us,scenario_043,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It recognized Colorado's FAMLI employee premium but then dropped it as 'state UI-like' and submitted FICA only ($312.73). The reference withholds the 0.44% employee share ($17.99). The other reading, where the employer pays that share, is not ruled out by the prompt."
+us,scenario_043,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It stated that Colorado has no mandatory employee payroll tax and submitted FICA only ($312.73). That leaves out the 0.44% FAMLI employee premium ($17.99), which the reference treats as withheld because the prompt doesn't say the employer pays it."
+us,scenario_043,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It computed FICA only ($312.73, rounded to $313) and never considered Colorado's 0.44% FAMLI employee premium ($17.99) on $4,088 of wages. Its answer matches the reading in which the employer covers the employee's FAMLI share."
+us,scenario_043,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA to $4,088 ($312.73) and left out Colorado's 0.44% FAMLI employee premium ($17.99). The reference withholds that premium. The prompt doesn't say whether the employer pays it."
+us,scenario_043,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,It stated that Colorado has no mandatory employee-side payroll tax and submitted FICA only ($312.73). It missed the FAMLI employee premium of 0.44% of wages ($17.99) in the reference. The only reading that supports leaving it out is that the employer pays it.
+us,scenario_043,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It looked only for state disability or unemployment-insurance employee taxes and concluded that Colorado has none. That misses the FAMLI employee premium (0.44%, $17.99) in the reference. Its FICA-only $312.73 matches the reading in which the employer pays that share."
+us,scenario_043,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It stated that Colorado has no employee state payroll tax and gave FICA only ($312.73), leaving out the 0.44% FAMLI employee premium ($17.99). The prompt doesn't say whether the employer covers that premium."
+us,scenario_043,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It correctly added the Colorado FAMLI employee premium but charged $24.53, which is 0.6% of $4,088, instead of the 2026 employee rate of 0.44% ($17.99). That overstated the total at $337.26 instead of $330.73."
+us,scenario_043,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It subtracted the $8,389 employer-sponsored insurance premium from wages as a pre-tax reduction to FICA wages and so reached zero payroll tax. The prompt reports gross wages of $4,088 and never says the premium is paid through a Section 125 salary reduction. FICA and FAMLI therefore apply to the full $4,088, for $330.73."
+us,scenario_043,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It applied only 7.65% FICA to $4,088 ($312.73), leaving out Colorado's 0.44% FAMLI employee premium ($17.99) in the reference. This is the reading in which the employer pays the employee's FAMLI share."
+us,scenario_043,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It added Social Security ($253.46) and Medicare ($59.28) for $312.74 but left out the Colorado FAMLI employee premium of 0.44% on $4,088 ($17.99). The prompt doesn't say whether the employer pays that premium."
+us,scenario_043,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It counted only Social Security and Medicare ($312.73), leaving out Colorado's 0.44% FAMLI employee premium ($17.99) in the reference. Its answer matches the reading in which the employer pays that share."
+us,scenario_043,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,It gave FICA only ($312.73) and never addressed the Colorado FAMLI employee premium of 0.44% of wages ($17.99) in the reference. The prompt doesn't say whether the employer covers that premium.
+us,scenario_043,payroll_tax,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"It applied 7.65% FICA to $4,088 ($312.73) and left out the Colorado FAMLI employee premium (0.44%, $17.99). This matches the reading in which the employer pays the employee's half of the 0.88% premium."
+us,scenario_043,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"It computed 7.65% FICA ($312.73, rounded to $313) and left out Colorado's 0.44% FAMLI employee premium ($17.99). The prompt doesn't say whether that share is withheld from the employee or paid by the employer."
+us,scenario_043,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It stated that Colorado has no state employee payroll tax, left out the 0.44% FAMLI employee premium ($17.99), and truncated FICA of $312.73 to $312. Its answer matches the reading in which the employer pays the FAMLI share."
+us,scenario_043,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It treated the $4,088 in wages as not subject to payroll tax and returned zero. FICA wages have no minimum: 6.2% Social Security and 1.45% Medicare apply from the first dollar of wages, plus the 0.44% FAMLI premium, for $330.73."
+us,scenario_043,payroll_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"It explicitly left out any Colorado employee payroll tax and gave FICA only ($312.73). The reference adds the 0.44% FAMLI employee premium ($17.99), and the prompt doesn't say whether the employer covers it."
+us,scenario_043,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It computed only 7.65% FICA ($312.73) and never accounted for Colorado's 0.44% FAMLI employee premium ($17.99) on $4,088 of wages. This is the reading in which the employer pays that share."
+us,scenario_043,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"It gave FICA only ($312.73, rounded to $313) and left out the Colorado FAMLI employee premium of 0.44% ($17.99) in the reference. The prompt doesn't say whether the employer covers that premium."
+us,scenario_043,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"It stated that Colorado has no employee payroll tax and submitted FICA only ($312.73). It missed the 0.44% FAMLI employee premium ($17.99), which the reference withholds because the prompt never says the employer pays it."
+us,scenario_043,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"It stated that Colorado has no employee payroll tax and gave FICA only ($312.73), leaving out the 0.44% FAMLI employee premium ($17.99). Its answer matches the reading in which the employer covers that share."
+us,scenario_043,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It stated that no mandatory state payroll tax applies and gave FICA only ($312.73, rounded to $313). That leaves out Colorado's 0.44% FAMLI employee premium ($17.99). The prompt doesn't say whether the employer pays it."
+us,scenario_043,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"It computed Social Security and Medicare only ($312.73) and never considered the Colorado FAMLI employee premium of 0.44% on $4,088 ($17.99). This matches the reading in which the employer pays the employee's FAMLI share."
+us,scenario_043,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"It stated that Colorado has no mandatory employee state payroll tax and submitted FICA only ($312.74, rounded to $313). That leaves out the 0.44% FAMLI employee premium ($17.99) in the reference. The prompt doesn't say who pays that share."
+us,scenario_043,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"It counted only Social Security and Medicare ($312.73), leaving out Colorado's 0.44% FAMLI employee premium ($17.99) on wages. Its answer matches the reading in which the employer covers that share."
+us,scenario_043,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It stated that Colorado has no mandatory employee state payroll tax and gave FICA only ($312.73), missing the 0.44% FAMLI employee premium ($17.99). The prompt doesn't say whether the employer pays that premium."
+us,scenario_043,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It ruled out any Colorado employee payroll tax and submitted FICA only ($312.73). It left out the FAMLI employee premium (0.44% of $4,088 = $17.99) that the reference withholds because the prompt doesn't say the employer pays it."
us,scenario_043,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_043,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly found zero net income and the full maximum allotment, but it used $291 a month, the FY2024 one-person maximum, instead of the FY2026 figure of $298. That gives $3,492 instead of $3,576."
-us,scenario_043,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model denied SNAP by applying a $2,250 non-elderly asset limit to $2,800 of savings. Because the head is 66, the elderly asset limit applies, and the household passes it. Colorado's TANF non-cash categorical eligibility also qualifies the household, so it receives the $298 monthly maximum."
-us,scenario_043,snap,claude-haiku-5.5,llm_error,asset_resource,False,"The model claimed the elderly asset and income tests and take-up rules produce zero. The prompt says to assume take-up, $2,800 is under the elderly asset limit, and gross income is only about 30% of the poverty guideline. The household therefore gets the full $298 monthly maximum."
-us,scenario_043,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly reached zero net income and a maximum of about $300 a month, then arbitrarily replaced it with $200 a month. With zero net income, the benefit is the full one-person maximum of $298 a month, or $3,576 a year."
-us,scenario_043,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model stated the maximum was about $292 and that net income was very low, then cut the benefit to an unsupported $192 a month. With net income at zero, the benefit equals the full $298 one-person maximum allotment."
-us,scenario_043,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model found near-zero net income but treated $203 a month as the maximum allotment. The FY2026 one-person maximum for the contiguous states is $298 a month, and with zero net income the household receives that full amount."
-us,scenario_043,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The eligibility and zero-net-income logic is right, but the model used $292, the FY2025 one-person maximum allotment, instead of $298. That gives $3,504 instead of $3,576."
-us,scenario_043,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly reached zero net income but overstated the one-person maximum allotment as $328 a month. The correct maximum is $298 a month, or $3,576 a year."
-us,scenario_043,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model correctly found the household eligible with zero net income but used $301 a month as the 2026 one-person maximum. The correct maximum allotment is $298 a month, or $3,576 a year."
-us,scenario_043,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model explicitly used the FY2025 maximum allotment of $292 a month for a 2026 benefit. The FY2026 one-person maximum is $298, which gives $3,576."
-us,scenario_043,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model correctly gave the maximum allotment but used $292 a month, the FY2025 figure, instead of the FY2026 one-person maximum of $298. That gives $3,504 instead of $3,576."
-us,scenario_043,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"The model correctly gave the maximum allotment but used $299 a month instead of $298, the FY2026 one-person maximum for the contiguous states. That makes the annual figure $12 too high."
-us,scenario_043,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly found zero net income but used $292 a month, the FY2025 maximum, instead of the FY2026 one-person maximum allotment of $298. That gives $3,504 instead of $3,576."
-us,scenario_043,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model asserted without support that income and assets disqualify the household. Gross income is about 30% of the poverty guideline, $2,800 passes the elderly asset test, and Colorado's TANF non-cash categorical eligibility applies, so the household receives the $298 monthly maximum."
-us,scenario_043,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly found zero net income but used $291 a month, the FY2024 one-person maximum, instead of the FY2026 figure of $298. That gives $3,492 instead of $3,576."
-us,scenario_043,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly found categorical eligibility and zero net income but gave a rounded $3,500 instead of computing 12 times the $298 FY2026 one-person maximum allotment, which is $3,576."
-us,scenario_043,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model gave zero with no derivation. This elderly one-person household has $387 a month of income, has $2,800 in assets (under the elderly limit), and is categorically eligible in Colorado, so it receives the full $298 monthly maximum."
-us,scenario_043,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The deduction logic leading to zero net income is right, but the model used $292 a month, the FY2025 maximum, instead of the FY2026 one-person maximum of $298. That gives $3,504 instead of $3,576."
-us,scenario_043,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model correctly gave the maximum allotment, but its $3,504 reflects the FY2025 figure of $292 a month instead of the FY2026 one-person maximum of $298 a month, which gives $3,576."
-us,scenario_043,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,The model correctly found zero net income but used $292 a month instead of the FY2026 one-person maximum allotment of $298. That understates the annual benefit by $72.
-us,scenario_043,snap,gemini-4-argon,llm_error,thresholds_rates,False,"The model correctly applied the earned-income, standard, and elderly medical deductions to reach zero net income, but used the stale $292 maximum instead of the FY2026 one-person maximum of $298."
-us,scenario_043,snap,glm-5.2,llm_error,thresholds_rates,False,"The model correctly gave the full one-person maximum but used about $302 a month ($3,624 a year). The FY2026 maximum allotment for the contiguous states is $298 a month, or $3,576 a year."
-us,scenario_043,snap,glm-5.3,llm_error,thresholds_rates,False,"The model correctly reached zero net income but overstated the one-person maximum allotment as $305 a month. The FY2026 figure is $298 a month, or $3,576 a year."
-us,scenario_043,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave about $106 a month with no derivation. After the earned-income, standard, and elderly excess-medical deductions, net income is zero, so the benefit is the full $298 monthly maximum, or $3,576 a year."
-us,scenario_043,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed the household lacks low-income indicators even though its $387 a month of gross income is about 30% of the poverty guideline. With an elderly head, $2,800 in assets, and Colorado categorical eligibility, the household receives the $298 monthly maximum."
-us,scenario_043,snap,grok-4.3,llm_error,asset_resource,False,"The model said $2,800 exceeds typical asset limits, but it ignored that the 66-year-old head qualifies for the higher elderly asset limit, which $2,800 passes. It also ignored that Colorado's TANF non-cash categorical eligibility applies, so the household receives the $298 monthly maximum."
-us,scenario_043,snap,grok-4.5,llm_error,thresholds_rates,False,"The eligibility and zero-net-income logic is correct, but the model used $292 a month, the FY2025 maximum, instead of the FY2026 one-person maximum allotment of $298."
-us,scenario_043,snap,grok-4.6,llm_error,thresholds_rates,False,"The model correctly found zero net income and gave the maximum allotment, but used the outdated $292 one-person maximum instead of the FY2026 figure of $298 a month."
-us,scenario_043,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model correctly gave the full allotment but used an approximate $300 a month instead of the FY2026 one-person maximum of $298. That gives $3,600 instead of $3,576."
-us,scenario_043,snap,inkling,llm_error,thresholds_rates,False,"The model correctly reached zero net income but rounded the one-person maximum allotment to $300 a month. The FY2026 figure is $298 a month, or $3,576 a year."
-us,scenario_043,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for SNAP, so there is no answer to compare with the $3,576 annual maximum-allotment benefit."
-us,scenario_043,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model correctly noted that the head is elderly and has low income, then gave zero without any calculation. The household passes the elderly asset test and the net income test with zero net income, so it receives the $298 monthly maximum, or $3,576 a year."
-us,scenario_043,snap,muse-spark-1.3,llm_error,thresholds_rates,False,"The eligibility logic is correct, but the model used $292 a month, the FY2025 maximum, instead of the FY2026 one-person maximum of $298. That gives $3,504 instead of $3,576."
-us,scenario_043,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model treated employer-sponsored insurance premiums as a shelter item and used an invented $243 monthly maximum. The household's net income is zero after the elderly medical deduction, so it receives the full $298 one-person maximum allotment."
-us,scenario_043,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted without derivation that no positive benefit results. The household passes the elderly asset test, is categorically eligible in Colorado, and has zero net income, so the benefit is $298 every month, or $3,576 a year."
+us,scenario_043,snap,claude-fable-5,llm_error,thresholds_rates,False,"It correctly got net income to $0 using the earned-income, standard and excess medical deductions. It then applied the outdated FY2024 one-person maximum allotment of $291 instead of the FY2026 figure of $298, giving $3,492 instead of $3,576."
+us,scenario_043,snap,claude-fable-5.2,llm_error,thresholds_rates,False,"It correctly used $298 a month for January–September. It then assumed an uprated $304 maximum for October–December, but the one-person maximum stays at $298 in those months, so the correct total is 12 × $298 = $3,576."
+us,scenario_043,snap,claude-haiku-4.5,llm_error,asset_resource,False,"It applied the $2,250 non-elderly asset limit and concluded that $2,800 in bank assets disqualifies the household. Because the head is 66, the elderly asset limit applies and the $2,800 passes it; the household is also categorically eligible through Colorado's BBCE. Net income is $0, so the household receives the $298 monthly maximum."
+us,scenario_043,snap,claude-haiku-5.5,llm_error,asset_resource,False,"It claimed the asset and income tests and a lack of take-up produce a zero benefit. In fact the $2,800 of assets passes the elderly asset test, the prompt requires assuming take-up, and the deductions bring net income to $0, so the benefit is $298 × 12 = $3,576."
+us,scenario_043,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It correctly found net income below zero and a maximum allotment of about $300. It then arbitrarily replaced that with $200 a month. With net income at $0, the benefit equals the full $298 maximum each month, or $3,576 a year."
+us,scenario_043,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"It acknowledged that net income is very low and the maximum is about $292, then cut the benefit to an unexplained $192 a month. Net income is $0, so the full one-person FY2026 maximum of $298 a month applies, or $3,576 a year."
+us,scenario_043,snap,claude-opus-5,llm_error,thresholds_rates,False,"It correctly derived near-zero net income but used about $203 a month as the one-person maximum allotment. The FY2026 contiguous-states maximum for one person is $298, which gives $3,576."
+us,scenario_043,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly reduced net income to $0 but used the FY2025 one-person maximum of $292. The engine applies the FY2026 maximum of $298 in every month of 2026, giving $3,576."
+us,scenario_043,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It correctly found zero net income but overstated the one-person maximum allotment at about $328 a month. The correct figure is $298, which yields $3,576 rather than $3,936."
+us,scenario_043,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It correctly found zero net income and passed the elderly asset test, but used $301 a month as the FY2026 one-person maximum allotment. The actual maximum is $298, giving $3,576."
+us,scenario_043,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"By its own statement it used the FY2025 maximum allotment of $292. For tax year 2026 the one-person maximum is $298 a month, giving $3,576."
+us,scenario_043,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly paid the maximum allotment to this zero-net-income elderly household but used the FY2025 figure of $292. The 2026 one-person maximum is $298, giving $3,576."
+us,scenario_043,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It correctly found negative net income but cited $299 a month as the FY2026 one-person maximum. The correct maximum is $298, which yields $3,576 instead of $3,588."
+us,scenario_043,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It correctly reduced net income to $0 using the elderly medical deduction but applied the FY2025 maximum of $292 to 2026. The one-person maximum for 2026 is $298, giving $3,576."
+us,scenario_043,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It asserted without support that the household's assets and income disqualify it. In fact $387 a month is about 30% of the poverty guideline, the household is categorically eligible through Colorado's BBCE, and $2,800 is under the elderly asset limit. Net income is $0, so the benefit is $298 × 12 = $3,576."
+us,scenario_043,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly got net income to $0 but used the FY2024 one-person maximum of $291. The 2026 figure is $298, giving $3,576 rather than $3,492."
+us,scenario_043,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It correctly found BBCE eligibility and zero net income but submitted a rounded $3,500, which works out to about $292 a month. The one-person maximum is $298 a month, giving $3,576."
+us,scenario_043,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 citing unspecified income and resource rules, but the household passes every test. Income is about 30% of the poverty guideline, Colorado BBCE applies, and the $2,800 of assets is under the elderly limit. Its deductions bring net income to $0, so the benefit is the $298 monthly maximum, or $3,576."
+us,scenario_043,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its deduction math (excess medical deduction of $381.67 a month, net income $0) is right, but it applied the FY2025 maximum of $292. The 2026 one-person maximum is $298, giving $3,576."
+us,scenario_043,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"It correctly found zero net income, but its $3,504 total implies the FY2025 maximum of $292 a month. The 2026 one-person maximum is $298, giving $3,576."
+us,scenario_043,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It correctly reduced net income to $0 but used the FY2025 maximum of $292 a month. For 2026 the one-person maximum is $298, giving $3,576."
+us,scenario_043,snap,gemini-4-argon,llm_error,thresholds_rates,False,"It correctly applied the earned-income, standard and elderly medical deductions to reach $0 net income, then used the outdated $292 maximum. The 2026 one-person maximum is $298, giving $3,576."
+us,scenario_043,snap,glm-5.2,llm_error,thresholds_rates,False,"It correctly found zero net income, but its $3,624 total implies a $302 monthly maximum allotment. The correct one-person maximum is $298, giving $3,576."
+us,scenario_043,snap,glm-5.3,llm_error,thresholds_rates,False,"It correctly reached zero net income but overestimated the one-person maximum at about $305 a month. The correct figure is $298 a month, giving $3,576."
+us,scenario_043,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $1,276 (about $106 a month) treats the household as receiving a heavily reduced benefit. After the 20% earned-income deduction, the $209 standard deduction and the elderly excess medical deduction of about $382 a month, net income is $0, so the full $298 maximum is payable every month."
+us,scenario_043,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It denied eligibility despite income of about 30% of the poverty guideline, elderly assets under the limit, and Colorado BBCE categorical eligibility. The household receives the $298 one-person maximum each month, or $3,576."
+us,scenario_043,snap,grok-4.3,llm_error,asset_resource,False,"It claimed $2,800 of assets exceeds the limits and ignored the listed $5,000 of medical expenses. The $2,800 is under the elderly asset limit, and the $381.67 monthly excess medical deduction brings net income to $0, so the benefit is $298 × 12 = $3,576."
+us,scenario_043,snap,grok-4.5,llm_error,thresholds_rates,False,"It correctly passed all tests and reached zero net income but used the FY2025 maximum of $292 a month. The 2026 one-person maximum is $298, giving $3,576."
+us,scenario_043,snap,grok-4.6,llm_error,thresholds_rates,False,"It correctly derived $0 net income but multiplied the outdated $292 maximum by 12. The one-person maximum applicable throughout 2026 is $298, giving $3,576."
+us,scenario_043,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It correctly reached zero net income but used an approximate $300 maximum allotment. The exact one-person contiguous-states maximum is $298, giving $3,576 rather than $3,600."
+us,scenario_043,snap,inkling,llm_error,thresholds_rates,False,"It correctly found zero net income but rounded the one-person maximum to about $300 a month. The exact figure is $298, giving $3,576."
+us,scenario_043,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no SNAP value and no explanation, so there was no answer to score. The correct derivation gives net income of $0 and the $298 one-person maximum each month, or $3,576."
+us,scenario_043,snap,minimax-m3,llm_error,categorical_eligibility,False,"It acknowledged that the elderly household meets more lenient limits but still asserted a $0 benefit without computing deductions. Income of about $387 a month, minus the earned-income, standard and excess medical deductions, leaves net income of $0, so the household receives the $298 maximum each month."
+us,scenario_043,snap,muse-spark-1.3,llm_error,thresholds_rates,False,"It correctly passed the income and asset tests and reached zero net income but used the FY2025 maximum of $292. The 2026 one-person maximum is $298, giving $3,576."
+us,scenario_043,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It treated the ESI premiums as a shelter expense and never finished the net-income computation. It then used an invented $243 monthly maximum. With net income at $0 after the excess medical deduction, the one-person maximum of $298 applies, giving $3,576."
+us,scenario_043,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"It asserted with no derivation that no positive benefit results. The household is categorically eligible through Colorado BBCE, passes the elderly asset test, and has $0 net income after the excess medical deduction, so the benefit is $298 × 12 = $3,576."
us,scenario_043,ssi,claude-fable-5.1,llm_error,period_annualization,False,"The model annualized the monthly SSI federal benefit rate and monthly income exclusions, then mechanically subtracted annual countable income to produce $9,954. It also identified that $2,800 exceeds the stated $2,000 resource limit but discarded that result; the controlling engine computation finds the individual’s approximately $4,642 of income above the SSI income limit, yielding zero."
us,scenario_043,ssi,claude-opus-4.7,llm_error,period_annualization,False,"The model converted annual wages and pension income into a monthly countable-income figure and subtracted it from an estimated monthly federal benefit rate, bypassing the SSI income-eligibility gate. The individual’s approximately $4,642 of income exceeds the applicable limit, so aged status and assumed take-up do not produce a payable benefit."
us,scenario_043,ssi,claude-opus-5,llm_error,period_annualization,False,"The model first annualized the SSI benefit rate and exclusions to derive about $9,360, then replaced that calculation with an unsupported $6,720 figure. Both paths miss that the individual’s approximately $4,642 of income fails the SSI income-limit test, which eliminates the benefit entirely."
us,scenario_043,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, so it failed the required structured-output contract."
-us,scenario_043,state_refundable_credits,claude-fable-5.1,llm_error,age_disability,False,"The model assumed Colorado's EITC covers childless workers aged 65 and older who are shut out federally only by age. Under PolicyEngine's Colorado rules, the state EITC is a percentage of the federal EITC actually allowed. The head is 66, above the federal 25–64 childless age window, so the federal EITC is $0 and 25% of $0 is $0, not 25% of the $312.73 phase-in amount."
-us,scenario_043,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model paid a $1,130 lowest-tier TABOR sales tax refund without checking the surplus trigger. C.R.S. 39-22-2003(2) pays that refund for a tax year only if the fiscal year ending in it has revenue above the Referendum C cap. FY2025-26 was forecast below the cap, so the tax year 2026 refund is $0."
-us,scenario_043,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model made up a $225 'senior/family affordability' refundable amount without naming a credit or a formula. The Family Affordability Credit requires a child under 17, and the senior housing and sales tax refunds are TABOR-surplus mechanisms that pay nothing in 2026. That leaves this childless senior with $0 in Colorado refundable credits."
-us,scenario_043,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated the Senior Housing Income Tax Credit and the TABOR sales tax refund as guaranteed flat payments to low-income seniors and estimated $1,200. Both are paid only when Colorado has excess revenue under the Referendum C cap, and FY2025-26 had none, so neither pays for tax year 2026. The Colorado EITC is also $0 because a 66-year-old childless filer gets no federal EITC."
-us,scenario_043,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"The model took 25% of a $625 federal EITC without applying the federal childless-EITC age ceiling of 64. At 66 the head has no federal EITC, so the Colorado match is $0. The $625 figure also did not come from the head's $4,088 of earnings, since the phase-in amount at that income is about $313."
-us,scenario_043,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model gave the head the maximum $800 Senior Housing Income Tax Credit just because the head is a low-income senior. That credit is funded through the TABOR surplus refund mechanism and pays nothing when the fiscal year ending in the tax year is under the Referendum C cap, which is the case for 2026. Colorado refundable credits are therefore $0."
-us,scenario_043,state_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"The model computed a $313 federal childless EITC (7.65% of $4,088) and took 50% of it. It ignored that federal childless EITC eligibility ends at age 64. The 66-year-old head gets no federal EITC, so the Colorado EITC is $0, and the 50% match rate is also wrong for 2026."
+us,scenario_043,state_refundable_credits,claude-fable-5.1,llm_error,age_disability,False,"The model claimed Colorado's EITC covers childless workers aged 65 and over who are excluded federally only because of age. It then applied a 25% match to a phase-in amount of 7.65% × $4,088. Colorado's EITC is a straight percentage of the federal EITC, and the federal childless EITC is $0 because the head is 66, above the age limit of 64. So the Colorado EITC is $0, and the total is $0."
+us,scenario_043,state_refundable_credits,claude-fable-5.2,llm_error,state_local_rule,False,"The model carried forward the 2024 lowest-tier TABOR sales tax refund ($177) into 2026. That refund is paid only when Colorado has excess revenue in the fiscal year ending in the tax year. FY2025-26 is forecast below the Referendum C cap, so the 2026 refund is $0."
+us,scenario_043,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model treated the TABOR sales tax refund as an automatic refundable credit for any full-year resident who files, and assigned an estimated $1,130 lowest-tier amount. It ignored the C.R.S. 39-22-2003(2) condition that the refund is paid only when the state has excess revenue. With FY2025-26 forecast below the Referendum C cap, no 2026 refund is paid and the total is $0."
+us,scenario_043,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model assigned a $225 refundable amount based on vague Colorado 'senior housing/income-qualified' credits. None of these Colorado refundable credits pays this household in 2026. The TABOR sales tax refund is $0 because there is no FY2025-26 revenue surplus, and the head has no Colorado EITC or child-based credits. The total is $0."
+us,scenario_043,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model assumed a flat senior-housing or TABOR-style sales tax refund of about $1,200 for a low-income 65+ filer. The TABOR sales tax refund is conditional on excess revenue, and FY2025-26 is forecast below the Referendum C cap, so it is $0 for 2026. No Colorado senior refundable credit pays this household either. The model also left room for a Colorado EITC, which is $0 because the 66-year-old head fails the federal childless EITC age limit of 64."
+us,scenario_043,state_refundable_credits,glm-5.3,llm_error,age_disability,False,"The model applied a 25% Colorado match to a $625 federal EITC. The head is 66, above the childless EITC age range of 25–64, so the federal EITC is $0 and the Colorado match is $0. The $625 base was also wrong even without the age bar: it is roughly the childless maximum credit, not 7.65% of the $4,088 in earnings."
+us,scenario_043,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model awarded the maximum Colorado Senior Housing Income Tax Credit of $800. That credit does not pay this household for tax year 2026, so Colorado's 2026 refundable credits for this head total $0. That total includes the TABOR refund, which is $0 because there is no revenue surplus."
+us,scenario_043,state_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"The model computed a $313 federal childless EITC and applied a 50% Colorado match. It missed that the 66-year-old head is above the federal childless EITC age limit of 64, so the federal EITC is $0 and the Colorado EITC, a percentage of the federal credit, is also $0."
us,scenario_043,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_044,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model took the full $56,806 of Social Security, subtracted the standard deduction, and taxed the remainder. It never applied the §86 provisional-income test: half of benefits, $28,403, is below the $32,000 joint base amount, so no benefits are includable and tax is $0."
us,scenario_044,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model applied only the older-couple standard deduction and nonrefundable credits, and its $4,162 answer only works if a large share of the Social Security counts as taxable income. It skipped the benefit-taxation test. Provisional income of $28,403 is below the $32,000 MFJ base amount, so taxable income and tax are both $0."
@@ -4538,53 +4593,54 @@ us,scenario_048,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,Fals
us,scenario_048,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled $34,626 of annual wages and Social Security as “very low countable income” and assigned an estimated $225 monthly benefit without applying the SNAP income threshold. That income exceeds the $17,420 limit for a one-person household, producing $0."
us,scenario_048,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model applied Ohio's 2.75% rate to the first dollar of taxable income, ignoring Ohio's zero-tax bracket: Ohio's rate schedule imposes no tax on Ohio taxable nonbusiness income at or below the roughly $26,050 threshold, and this filer's $7,656 of wages (Social Security survivor benefits are excluded from federal AGI here because provisional income of $21,141 is under the $25,000 base amount) falls far below it, yielding $0. It compounded the error by inventing a nonexistent $1,400 Ohio 'single standard deduction' — Ohio starts from federal AGI and allows personal exemptions, not a standard deduction — and then taxed the residual $3,856 at 2.75% to reach $106."
us,scenario_048,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The model taxed the entire $7,656 of Ohio taxable income at 2.75% ($210.54), skipping Ohio's zero-rate first bracket that exempts all Ohio taxable nonbusiness income up to about $26,050, under which this household's liability is $0 before any credit is considered. It also mis-specified the personal exemption as a credit equal to 2.75% of $2,400 ($66) — Ohio subtracts the $2,400 exemption from Ohio adjusted gross income and separately allows only a $20-per-exemption credit — and then netted a $50 senior citizen credit against a tax that never existed, landing on $94.54."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own working got to $30,830, but it left out the $580 non-itemizer cash charitable deduction (taxable income $190,185 instead of $189,605). It then submitted $33,223, a number that does not match its own arithmetic."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It never took the $25,000 qualified-overtime deduction or the $580 non-itemizer charitable deduction. It also deducted the $721 traditional IRA contribution, which is phased out, and used the outdated $29,200 standard deduction instead of $32,200. Even on its own $217,364 taxable income, $48,837 is far above what the 2026 MFJ brackets produce."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It answered $0 without computing anything. AGI of $247,385, minus the $32,200 standard deduction, the $25,000 overtime deduction and the $580 charitable deduction, leaves $189,605 of taxable income and $30,703 of tax."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction and the non-itemizer charitable deduction, used a $31,500 standard deduction instead of $32,200, and added Additional Medicare Tax, which belongs in payroll tax, not income tax. It then submitted $49,861, which does not match its own $36,571."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction and the $580 non-itemizer charitable deduction, and used a $30,300 standard deduction instead of $32,200. It then submitted $53,503, which does not match its own $37,055."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It counted the roughly $40,000 SALT cap as the deduction itself, not the $20,655 of real estate taxes actually paid. That inflated itemized deductions to $46,081, when the allowable total is about $24,813 after the mortgage debt cap and the charitable deduction floor, below the $32,200 standard deduction. It also left out the $25,000 overtime deduction, and its $47,500 overstates tax even on its own taxable income."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It never applied the $25,000 qualified-overtime deduction or the $580 non-itemizer charitable deduction. It also used the 2025 $30,000 standard deduction and 2025 brackets, not the 2026 figures ($32,200; 22% bracket starting at $100,800)."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction and the charitable deduction, and deducted the $721 IRA contribution, which is phased out. It then pushed its own $38,831 estimate up to $62,700 with no stated basis, partly by mixing in Additional Medicare Tax."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It deducted the $928 auto-loan interest, but that deduction drops by $200 for every $1,000 of MAGI over $200,000, so at $247,385 it is zero. It also left out the $580 non-itemizer charitable deduction. Net, its taxable income was $348 too low, which leaves the tax $76.59 short."
-us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction and the $580 non-itemizer charitable deduction, and used a $31,000 standard deduction instead of $32,200. That taxed about $26,800 of extra income at 22%."
-us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used pre-TCJA 2026 law (10/15/25/28% brackets, a $15,600 standard deduction, uncapped SALT itemizing). Current law has the $32,200 standard deduction, the 2026 TCJA-style brackets and the $25,000 overtime deduction, which it never took. It also excluded the $16,408 ESI premiums from wages, though only the 401(k) deferral reduces them."
-us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It excluded the ESI premiums from wages, though only the $15,436 401(k) deferral reduces them, and it never applied the $25,000 qualified-overtime deduction. Its $36,639 is in line with taxing roughly $25,000 more income at 22% than the reference does."
-us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its AGI, standard deduction and brackets all match the reference, but it left out the $25,000 qualified-overtime deduction and the $580 non-itemizer charitable deduction. That left taxable income at $215,185 instead of $189,605."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA provisions had expired, so it used personal exemptions, uncapped SALT itemizing and 15/25% brackets. Under current 2026 law that means a $32,200 standard deduction plus the $25,000 overtime deduction, and it took neither. It also excluded the $16,408 ESI premiums from wages and submitted $37,261, which does not match its own $36,330."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It worked from about $229,000 of taxable income. The correct figure is $189,605 after the $32,200 standard deduction, the $25,000 overtime deduction and the $580 charitable deduction. Its $52,368 also exceeds what the 2026 MFJ brackets give even on $229,000."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (personal exemptions, uncapped SALT itemizing) and so missed the $32,200 standard deduction and the $25,000 qualified-overtime deduction. It also excluded the $16,408 ESI premiums from wages, though only the 401(k) deferral reduces them."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used post-TCJA-expiration schedules with $10,100 of personal exemptions and $26,736 of itemized deductions. Current 2026 law allows the $32,200 standard deduction plus the $25,000 overtime deduction and the $580 charitable deduction, which brings taxable income to $189,605."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $42,203 overstates the reference by about $11,500, which fits leaving out the $25,000 qualified-overtime deduction and making further base or bracket errors. The correct taxable income after the standard, overtime and charitable deductions is $189,605."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It excluded the $16,408 ESI premiums from wages, though only the 401(k) deferral reduces them, and used a $30,000 standard deduction instead of $32,200. It also never applied the $25,000 qualified-overtime deduction or the $580 non-itemizer charitable deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $37,921 corresponds to taxable income of about $222,015 under the 2026 MFJ brackets with $6,208 of qualified dividends taxed at 15%. The correct base is $189,605."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the 2025 $30,000 standard deduction and 2025 brackets instead of the 2026 $32,200 standard deduction and brackets. It also left out the $25,000 qualified-overtime deduction and the $580 non-itemizer charitable deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It claimed the $928 auto-loan-interest deduction, but that deduction is fully phased out: it drops $200 for every $1,000 of MAGI over $200,000, and MAGI is $247,385. The extra $928 deduction at 22% accounts for the whole $204 shortfall."
-us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,It gave no value and no explanation for federal_income_tax_before_refundable_credits.
-us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It took the $32,200 standard deduction but never applied the $25,000 qualified-overtime deduction or the $580 non-itemizer charitable deduction. That left taxable income at $215,185 instead of $189,605."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave no actual computation and treated the household as itemizing under a $10,000 SALT cap. Its $20,460 understates the tax: $189,605 of taxable income, with $6,208 of qualified dividends taxed at 15%, comes to $30,703 under the 2026 MFJ brackets."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $38,566 fits leaving out the $25,000 qualified-overtime deduction, which taxes about $25,000 more income at 22% than the correct $189,605 base."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It treated the $16,408 ESI premiums as a pre-tax salary reduction and cut AGI to $230,977. Only the $15,436 traditional 401(k) deferral reduces wages, so AGI is $247,385. Taxing the excluded $16,408 at 22% accounts for the whole $3,610 shortfall."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It claimed the qualified-vehicle loan-interest deduction, which is fully phased out: it drops $200 for every $1,000 of MAGI over $200,000, and MAGI is $247,385. The extra $928 deduction at 22% is the $204 shortfall."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It excluded the $16,408 ESI premiums from wages as a pre-tax deduction and arrived at taxable income of $173,197. Only the 401(k) deferral reduces wages, so taxable income is $189,605, and the excluded premiums account for the whole $3,610 shortfall."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It claimed the vehicle-loan-interest deduction, which is fully phased out above $200,000 MAGI. Its $33,473 also implies about $12,600 more taxable income at 22% than the correct $189,605 base, which means it did not take the full $25,000 overtime deduction it listed."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 ESI premiums from wages and arrived at AGI of $230,977. Only the $15,436 traditional 401(k) deferral reduces wages, so AGI is $247,385, and the excluded premiums taxed at 22% are the $3,610 gap."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It left out the $580 non-itemizer cash charitable deduction, which leaves taxable income at $190,185 instead of $189,605. At 22%, that is the $127.55 overstatement."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It treated the $16,408 ESI premiums as pre-tax and excluded them, cutting AGI to $230,977. Only the 401(k) deferral reduces wages, so AGI is $247,385 and taxable income is $189,605."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,"It treated the qualified-overtime deduction as phased out. The MFJ phase-out starts at $300,000 of MAGI and MAGI here is $247,385, so the full $25,000 is allowed. It also left out the $580 non-itemizer charitable deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It put AGI at about $263,000 without subtracting the $15,436 traditional 401(k) deferral, and itemized instead of taking the $32,200 standard deduction. It also left out the $25,000 qualified-overtime deduction, which inflated taxable income to about $230,000 when the correct figure is $189,605."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (uncapped property-tax itemizing plus $10,500 of personal exemptions) instead of current 2026 law, which has the $32,200 standard deduction and the $25,000 qualified-overtime deduction. That left taxable income at $210,149 instead of $189,605."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used post-TCJA-sunset rules: personal exemptions, uncapped itemizing and 10/15/25% brackets. It never applied the $32,200 standard deduction or the $25,000 qualified-overtime deduction, which left taxable income at $209,739."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It left out the $580 non-itemizer cash charitable deduction, which leaves taxable income at $190,185 instead of $189,605. It also mis-added the brackets: $183,977 of ordinary income comes to $29,899 under 2026 MFJ rates, not the $30,051 it reported."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It reasoned from capped itemized deductions and never took the $25,000 qualified-overtime deduction. Its $36,370 fits a taxable base about $25,000 above the correct $189,605."
-us,scenario_049,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its AGI of $230,256 comes from excluding the $16,408 ESI premiums and deducting the $721 IRA contribution, neither of which reduces income. It then left out the $25,000 qualified-overtime deduction and the $580 charitable deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA had expired and used personal exemptions, uncapped itemizing and pre-TCJA brackets. Current 2026 law has the $32,200 standard deduction and the $25,000 overtime deduction. It also excluded the $16,408 ESI premiums from wages."
-us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It capped the overtime deduction at $25,675 instead of the $25,000 MFJ cap, deducted the $721 IRA contribution (phased out), used a $30,900 standard deduction and left out the charitable deduction. It also taxed the full $190,089 at ordinary rates, qualified dividends included, and then added 15% on the $6,208 again, so those dividends were taxed twice."
-us,scenario_049,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used a $31,000 MFJ standard deduction instead of the 2026 $32,200, so taxable income came out at $190,805 instead of $189,605, and it also slightly mis-added the brackets."
-us,scenario_049,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It subtracted $4,800 of health insurance premiums from wages as pre-tax, but only the $15,436 traditional 401(k) deferral reduces wages, so AGI is $247,385, not $242,585. It also left out the $580 non-itemizer charitable deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It never applied the $25,000 qualified-overtime deduction or the $580 non-itemizer charitable deduction. It also deducted the $721 traditional IRA contribution, which is fully phased out for an active plan participant at $247,385 MAGI."
-us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It made a subtraction error ($262,821 − $15,436 = $247,385, not $258,821) and chose $15,395 of itemized deductions over the larger $32,200 standard deduction. It also left out the $25,000 overtime deduction and taxed all $6,320 of dividends as qualified. It then submitted $51,215, which does not match its own $42,933."
-us,scenario_049,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It deducted the $2,724 Roth 401(k) contribution alongside the traditional 401(k), used a $32,900 standard deduction and left out the $25,000 qualified-overtime deduction. Its $44,587 is also far above what the 2026 MFJ brackets give on its own $205,441 of ordinary income."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its own work reached $30,830 because it left out the 2026 non-itemizer cash-charity deduction ($580 claimed, $579.76 allowed by the reference), which is worth about $128 of tax. It then submitted $33,223, which does not match any step of its calculation."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,credit_phaseout,False,"It took the $721 traditional IRA deduction even though the spouse is covered by a 401(k). Joint MAGI of about $247k is far above the active-participant phaseout, so the IRA deduction is zero and AGI is $247,385. That extra $721 at 22% explains the $159 shortfall."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used the old $29,200 standard deduction instead of $32,200 and wrongly deducted the $721 IRA contribution. It also left out the $25,000 qualified-overtime deduction and the non-itemizer charity deduction. Even on its own $217,364 taxable income, $48,837 is far above what the 2026 joint brackets produce."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It reported $0 tax for a couple with $247,385 of AGI and $189,605 of taxable income. The correct ordinary tax is $29,771 plus $931 on qualified dividends."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction and the non-itemizer charity deduction, and used a $31,500 standard deduction instead of $32,200. It also added the Additional Medicare Tax, which is a payroll tax. It then submitted $49,861, which does not match its own $36,571 total."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It never applied the $25,000 qualified-overtime deduction or the non-itemizer charity deduction, and it used a $30,300 standard deduction and pre-OBBBA brackets. It then submitted $53,503, which does not match its own $37,055 result."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It counted the roughly $40,000 SALT cap as the deduction instead of the actual $20,655 of real estate taxes. That inflated itemized deductions to $46,081, when the real total of about $26,700 is below the $32,200 standard deduction. It also left out the $25,000 overtime deduction, and $47,500 overstates tax even on its own taxable income."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used the 2025 $30,000 standard deduction and 2025 brackets instead of the 2026 figures ($32,200; 22% bracket ending at $211,400). It also never subtracted the $25,000 qualified-overtime deduction or the $580 non-itemizer cash-charity deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It deducted the nondeductible $721 IRA contribution and left out the $25,000 overtime deduction and the charity deduction. It then arbitrarily raised its own $38,831 estimate to $62,700, citing Additional Medicare Tax, which is not part of income tax."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It subtracted the $928 auto-loan interest in computing AGI. That deduction does not reduce AGI, and it shrinks by $200 per $1,000 of MAGI over $200,000, so at this income it is zero. It also left out the $580 non-itemizer cash-charity deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It left out the $25,000 qualified-overtime deduction and the non-itemizer charity deduction. It also used a $31,000 standard deduction and wrong bracket thresholds instead of $32,200 and the 2026 schedule (10% to $24,800, 12% to $100,800)."
+us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied the expired pre-TCJA system (personal exemptions and 15%/25%/28% brackets), although OBBBA made the TCJA brackets and $32,200 standard deduction permanent. It also wrongly subtracted the $16,408 employer insurance premiums from wages and left out the $25,000 overtime deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It reduced wages by the employer-sponsored insurance premiums, but only the $15,436 401(k) deferral reduces wages, so AGI is $247,385. It also never applied the $25,000 qualified-overtime deduction or the non-itemizer charity deduction, which leaves its $36,639 well above the correct tax."
+us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It had the correct AGI, standard deduction and 2026 brackets, but left out the $25,000 qualified-overtime deduction and the $579.76 non-itemizer charity deduction. Taxing $215,185 instead of $189,605 produces exactly its $36,330.14."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (personal exemptions, uncapped SALT itemizing, 15%/25% brackets) instead of the permanent post-OBBBA law. It also subtracted the $16,408 employer insurance premiums from wages and left out the $25,000 overtime deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its taxable income of about $229,000 leaves out the $25,000 qualified-overtime deduction, the non-itemizer charity deduction and part of the $32,200 standard deduction. The correct taxable income is $189,605, and $52,368 also exceeds what the 2026 brackets produce."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used pre-TCJA 2026 rules (personal exemptions, unlimited real estate tax itemizing) even though OBBBA made the TCJA structure permanent. It also subtracted the employer insurance premiums from wages and never applied the $25,000 overtime deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied TCJA-expiration rules (itemized deductions plus $10,100 of personal exemptions, 25% bracket) instead of the permanent $32,200 standard deduction and 2026 brackets. It also left out the $25,000 qualified-overtime deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no calculation, and $42,203 matches neither the AGI of $247,385 nor taxable income of $189,605. The answer is consistent with skipping the $25,000 overtime deduction and overstating bracket tax."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 employer insurance premiums from wages and used the 2025 $30,000 standard deduction. It also left out the $25,000 qualified-overtime deduction and the non-itemizer charity deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It started from the correct income items but did not subtract the $25,000 qualified-overtime deduction or the non-itemizer charity deduction. That explains its $37,921, about $7,200 above the correct tax on $189,605 of taxable income."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the 2025 $30,000 standard deduction and 2025 brackets instead of $32,200 and the 2026 schedule. It also never subtracted the $25,000 qualified-overtime deduction or the $579.76 non-itemizer charity deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"It claimed the full $928 auto-loan interest deduction, which shrinks by $200 per $1,000 of MAGI over $200,000 for joint filers. With MAGI of $247,385 the reduction is $9,477, so the deduction is zero. That extra $928 at 22% explains the $204 shortfall."
+us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,It returned no value or reasoning for federal_income_tax_before_refundable_credits.
+us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It never applied the $25,000 qualified-overtime deduction or the non-itemizer charity deduction. It also overstated ordinary tax ($36,982 instead of $35,399 on $208,977 under the 2026 brackets)."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave no actual calculation, and $20,460 understates the tax on $189,605 of taxable income. The 2026 joint brackets give $29,771 of ordinary tax plus $931 on qualified dividends."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and $38,566 is consistent with taxing AGI after only a standard deduction. It left out the $25,000 qualified-overtime deduction and the non-itemizer charity deduction, which bring taxable income down to $189,605."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 employer-sponsored insurance premiums from wages, but only the $15,436 traditional 401(k) deferral reduces IRS employment income. AGI is therefore $247,385, not $230,977. The excluded $16,408 at 22% explains the $3,610 shortfall."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It took the qualified vehicle-loan interest deduction for the full $928. That deduction shrinks by $200 per $1,000 of MAGI over $200,000 for joint filers and is fully phased out at $247,385 MAGI."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $16,408 employer health premiums, but only the $15,436 traditional 401(k) deferral comes out of IRS employment income. That understated AGI by $16,408 and tax by about $3,610."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its answer is $2,770 above the correct tax, which equals about $12,600 of missing deductions at 22%. That matches using the $12,500 per-person overtime cap instead of the $25,000 joint cap, net of the vehicle-loan interest it wrongly claimed (that deduction is fully phased out at this MAGI)."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted the $16,408 employer health premiums from wages, giving AGI of $230,977 instead of $247,385; only the 401(k) deferral reduces IRS employment income. Its other deductions were correct, so the lost $16,408 at 22% accounts for the whole $3,610 gap."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It left out the 2026 non-itemizer cash-charity deduction ($579.76 of the $580 cash gift), which reduces taxable income to $189,605.23. Without it, taxable income stays at $190,185, which produces its $30,830.14."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It treated the $16,408 employer health premiums as excluded from wages, giving AGI of $230,977. The only pre-tax wage reduction is the $15,436 401(k) deferral, so AGI is $247,385."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,"It said the overtime deduction was phased out, but the phaseout starts only above $300,000 of MAGI for joint filers. At $247,385 MAGI the full $25,000 is allowed, and the $580 non-itemizer charity deduction also applies."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It used AGI of about $263k without subtracting the $15,436 traditional 401(k) deferral. It also itemized about $20k instead of taking the $32,200 standard deduction, and left out the $25,000 qualified-overtime deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (uncapped property tax itemizing plus $10,500 of personal exemptions) instead of the permanent $32,200 standard deduction and 2026 brackets. It also never applied the $25,000 qualified-overtime deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used post-sunset rules (personal exemptions and 10/15/25% brackets) although OBBBA made the TCJA structure permanent. It also left out the $25,000 qualified-overtime deduction and the non-itemizer charity deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It left out the $580 non-itemizer cash-charity deduction, so it taxed $190,185 instead of $189,605. It also overstated ordinary tax on $183,977 ($30,051 instead of $29,899 under the 2026 joint brackets)."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It never applied the $25,000 qualified-overtime deduction or the non-itemizer charity deduction. That leaves taxable income about $25,600 too high, which explains its $36,370."
+us,scenario_049,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Its AGI of $230,256 wrongly subtracts both the $16,408 employer insurance premiums and the nondeductible $721 IRA contribution. It also left out the $25,000 qualified-overtime deduction and the non-itemizer charity deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It applied expired-TCJA rules (personal exemptions, uncapped SALT itemizing, pre-TCJA brackets) and subtracted the $16,408 employer premiums from wages. It also left out the $25,000 qualified-overtime deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It ran all $190,089 of taxable income through the ordinary brackets, including the qualified dividends, and then added 15% on the $6,208 again, so the dividends were taxed twice. It also deducted the nondeductible $721 IRA contribution, set the overtime deduction at $25,675 instead of the $25,000 cap, and used a $30,900 standard deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used a $31,000 standard deduction instead of the 2026 joint amount of $32,200, so its taxable income of $190,805 is too high. The correct figure is $189,605.23."
+us,scenario_049,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It subtracted $4,800 of health insurance premiums from wages, giving AGI of $242,585. Only the $15,436 401(k) deferral reduces wages, so AGI is $247,385. It also left out the $580 non-itemizer cash-charity deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It deducted the $721 traditional IRA contribution, which is fully phased out for an active participant at this income. It also left out the $25,000 qualified-overtime deduction and the non-itemizer charity deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It mis-subtracted the 401(k) deferral and got AGI of $258,821 instead of $247,385. It then itemized $15,395 even though the $32,200 standard deduction is larger, and left out the $25,000 overtime deduction. It submitted $51,215, which does not match its own $42,933."
+us,scenario_049,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It deducted the nondeductible IRA contribution, used a $32,900 standard deduction, and left out the $25,000 overtime deduction. It also computed $44,587 of ordinary tax on $205,441, about $9,700 more than the 2026 joint brackets (10%/12%/22%) produce."
us,scenario_049,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_049,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_049,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -4705,57 +4761,58 @@ us,scenario_051,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The
us,scenario_051,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax value or explanation. It therefore failed the required structured-output contract.
us,scenario_051,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly derived $2,480 of Social Security tax plus $580 of Medicare tax and explicitly totaled them as $3,060, but submitted $3,074.95. The submitted value is inconsistent with every component in its explanation."
us,scenario_051,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,reference_later_law,taxable_income_or_deductions,False,"It used 2025's $12,500 deduction to calculate $830.40, then submitted $1,020 without supporting arithmetic. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_later_law,taxable_income_or_deductions,False,"It applied the 3% flat rate correctly but used the 2025 base standard deduction of $12,500. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_later_law,taxable_income_or_deductions,False,"It said Louisiana has no state individual income tax. Louisiana taxes individual income at a flat 3%. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_later_law,taxable_income_or_deductions,False,"It said the standard deduction and personal exemption wipe out all liability. Louisiana repealed the personal exemption from 2025, so income above the standard deduction stays taxable at the flat 3% rate. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_later_law,taxable_income_or_deductions,False,"It correctly used AGI of $40,180 and the 3% flat rate. It used 2025's $12,500 deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_later_law,taxable_income_or_deductions,False,"After computing $830.40, it subtracted a 'personal credit' that does not exist and got $632. Louisiana's reform repealed the personal exemption and credit, and no nonrefundable credit applies here. It also used 2025's $12,500 deduction. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,reference_later_law,taxable_income_or_deductions,False,"It started with the unindexed $12,500 deduction and then cut the result to $812 by an arbitrary 'rounding of federal AGI adjustments'. AGI is $40,180. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_later_law,taxable_income_or_deductions,False,"It used 2025's $12,500 deduction and calculated $830.40 on $27,680 of taxable income. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_later_law,taxable_income_or_deductions,False,"It subtracted about $2,783 of federal income tax from Louisiana income, but that deduction was discontinued in 2022. It used 2025's $12,500 standard deduction and calculated about $747. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_later_law,taxable_income_or_deductions,False,"It computed $830.40 with the unindexed $12,500 deduction, then submitted $1,080 with no supporting calculation. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_later_law,taxable_income_or_deductions,False,"It used a $12,500 standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_later_law,taxable_income_or_deductions,False,"It combined the 3% rate with two $4,500 offsets, leaving $31,180 taxable, and calculated $935.40 before rounding to $935. Louisiana repealed the $4,500 personal exemption from 2025 and applies its own standard deduction to this household. The retained reference is $820.35 on $27,345 of taxable income using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_later_law,taxable_income_or_deductions,False,"It used a 4.25% rate instead of the flat 3%. It also cut AGI to $34,791 by subtracting the $5,389 employer insurance premiums, which are not in the $40,000 gross wages, and it used the repealed $4,500 exemption. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_later_law,taxable_income_or_deductions,False,"It used 2025's $12,500 single standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_later_law,taxable_income_or_deductions,False,"It used 2025's $12,500 deduction and calculated $830.40 on $27,680 of taxable income. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_later_law,taxable_income_or_deductions,False,"It used the repealed pre-2025 system: the 1.85%/3.5% brackets, the deduction for federal income tax paid, and the $4,500 combined deduction. It also reduced AGI to $34,791 by subtracting employer insurance premiums. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_later_law,taxable_income_or_deductions,False,"It applied 'progressive' Louisiana brackets, but those were replaced by a flat 3% rate from 2025. Its $724 does not match 3% × ($40,180 − $12,835) = $820.35."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_later_law,taxable_income_or_deductions,False,"It used the repealed 1.85%/3.5% brackets and the $4,500 exemption, and it subtracted the $5,389 employer premiums to get AGI of $34,791. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_later_law,taxable_income_or_deductions,False,"It used the repealed pre-2025 structure: a deduction for federal income tax paid, a $4,500 exemption, and 1.85%/3.5% brackets. It also started from an understated AGI. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_later_law,taxable_income_or_deductions,False,"It gave $1,115 with no computation, which is more than even 3% of income after the unindexed deduction. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_later_law,taxable_income_or_deductions,False,"It applied the repealed 1.85%/3.5% brackets and the $4,500 combined deduction to an AGI of $34,791, which it got by subtracting employer insurance premiums. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_later_law,taxable_income_or_deductions,False,"It used the pre-reform $4,500 deduction and the 1.85%/3.5% brackets, which leaves $35,680 taxable. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_later_law,taxable_income_or_deductions,False,"It claimed a $4,500 single standard deduction and applied the 1.85%/3.5% brackets. From 2025 Louisiana taxes income at a flat 3% after its own standard deduction and allows no $4,500 offset. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-4-argon,reference_later_law,taxable_income_or_deductions,False,"It used 2025's $12,500 standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,reference_later_law,taxable_income_or_deductions,False,"It used the 3% rate and AGI of $40,180 with a $12,500 deduction. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,glm-5.3,reference_later_law,taxable_income_or_deductions,False,"It used a $12,500 single standard deduction and calculated $830.40 on $27,680 of taxable income. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_later_law,taxable_income_or_deductions,False,"It said deductions and exemptions eliminate all Louisiana tax. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_later_law,taxable_income_or_deductions,False,"It gave $781 with no traceable calculation, which implies about $26,033 of taxable income at 3%. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,reference_later_law,taxable_income_or_deductions,False,"It used 2025's $12,500 standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_later_law,taxable_income_or_deductions,False,"It used a $12,500 single standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_later_law,taxable_income_or_deductions,False,"It applied 3% to $27,680 using the unindexed $12,500 deduction. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_later_law,taxable_income_or_deductions,False,"It used a $12,500 single standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-astra,reference_later_law,taxable_income_or_deductions,False,"It used a $12,500 standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-luna,reference_later_law,taxable_income_or_deductions,False,"It used an estimated $12,500 deduction and calculated $830.40 on $27,680 of taxable income. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-sol,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 single deduction. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_later_law,taxable_income_or_deductions,False,"It used a $12,500 single standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-6.1-terra,reference_later_law,taxable_income_or_deductions,False,"It applied the unindexed $12,500 standard deduction. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,reference_later_law,taxable_income_or_deductions,False,"It gave $680 with no computation, which implies about $22,667 of taxable income at 3%. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,reference_later_law,taxable_income_or_deductions,False,"It subtracted the federal standard deduction instead of Louisiana's own standard deduction. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994. It also applied the repealed 1.85%/3.5% brackets instead of the flat 3% rate."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-4.6,reference_later_law,taxable_income_or_deductions,False,"It used a $12,500 standard deduction and rounded $830.40 to $830. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-4.7,reference_later_law,taxable_income_or_deductions,False,"It used 2025's $12,500 single standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,reference_later_law,taxable_income_or_deductions,False,"It claimed a federal-tax deduction and two $4,500 offsets and applied the 1.85%/3.5% brackets. Louisiana's federal-tax deduction ended in 2022, and from 2025 the state repealed the personal exemption and moved to a flat 3% rate. It also understated AGI at $34,791 by subtracting employer insurance premiums. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,inkling,reference_later_law,taxable_income_or_deductions,False,"It used a $12,500 standard deduction and rounded $830.40 to $830. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for this variable, so nothing could be scored. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,reference_later_law,taxable_income_or_deductions,False,"It used a $12,500 single standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,reference_later_law,taxable_income_or_deductions,False,"It claimed a $4,500 standard deduction, dropped the $180 capital gain, and computed $1,065. It then submitted $1,208, which its own reasoning does not support. Louisiana has allowed no $4,500 offset since 2025, when it repealed the personal exemption and applied its own standard deduction. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,muse-spark-1.3,reference_later_law,taxable_income_or_deductions,False,"It used a $12,500 standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,muse-wonder,reference_later_law,taxable_income_or_deductions,False,"It used 2025's $12,500 deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,ox-alpha,reference_later_law,taxable_income_or_deductions,False,"It used a $12,500 single standard deduction and calculated $830.40. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_later_law,taxable_income_or_deductions,False,"It subtracted the federal standard deduction and a repealed $4,500 exemption, then applied made-up 1.75%/3.55% brackets. It submitted $573 even though its own math gave $482.52. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
-us,scenario_051,state_income_tax_before_refundable_credits,qwen3.8-max,reference_later_law,taxable_income_or_deductions,False,"Its $1,324.80 equals 3% of $44,160, a base larger than the $40,180 AGI, so it did not subtract the standard deduction from a correct base. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,reference_later_law,taxable_income_or_deductions,False,"It applied the unindexed $12,500 single standard deduction instead of the 2026 inflation-indexed $12,835 and computed $830.40. It then submitted $1,020, which none of its own arithmetic supports. The correct figure is 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_later_law,taxable_income_or_deductions,False,"It used the base $12,500 single standard deduction without the inflation indexing that begins in 2026. The indexed 2026 deduction is $12,835, so taxable income is $27,345, not $27,680, and the tax is $820.35, not $830.40."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5.2,reference_later_law,taxable_income_or_deductions,False,"It correctly indexed the standard deduction for 2026 but overestimated the inflation adjustment, using $12,875 instead of $12,835. That understated taxable income by $40 ($27,305 instead of $27,345), giving $819.15 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_later_law,taxable_income_or_deductions,False,"It wrongly said Louisiana has no individual income tax. Louisiana taxes income at a flat 3% after a $12,835 single standard deduction in 2026, which gives $820.35 on $27,345 of taxable income."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_later_law,taxable_income_or_deductions,False,"It claimed the standard deduction and exemption wipe out the liability, but the only offset is the $12,835 single standard deduction. That leaves $27,345 taxable at 3%, or $820.35, not $0."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_later_law,taxable_income_or_deductions,False,"It cited Act 11's $12,500 single standard deduction but left out that act's inflation indexing starting in 2026. The 2026 deduction is $12,835, so the tax is 3% × $27,345 = $820.35 rather than $830.40."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_later_law,taxable_income_or_deductions,False,"It reduced its $830.40 by a made-up 'personal credit' to reach $632. The reform repealed the personal exemption, and no nonrefundable credit applies to this filer. It also used the unindexed $12,500 deduction instead of the 2026 indexed $12,835."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 deduction (giving $830) and then cut the result to $812 for unexplained 'rounding of federal AGI adjustments'. Louisiana AGI is exactly $40,180, and the indexed $12,835 deduction gives exactly $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_later_law,taxable_income_or_deductions,False,"It subtracted the base $12,500 single standard deduction instead of the 2026 inflation-indexed $12,835. That overstated taxable income by $335 and gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_later_law,taxable_income_or_deductions,False,"It deducted $2,783 of federal income tax from Louisiana income, but the 2025 reform repealed that deduction. It also used the unindexed $12,500 deduction instead of $12,835. Correct taxable income is $40,180 − $12,835 = $27,345, taxed at 3% for $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_later_law,taxable_income_or_deductions,False,"It computed $830.40 with the unindexed $12,500 deduction and then submitted $1,080 without support. The only adjustment is the indexed $12,835 standard deduction, and no credits apply, so the tax is $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_later_law,taxable_income_or_deductions,False,"It assumed the $12,500 single standard deduction carries over unchanged to 2026. The deduction is inflation-indexed to $12,835 in 2026, so taxable income is $27,345 and the tax is $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_later_law,taxable_income_or_deductions,False,"It paired the new 3% flat rate with the repealed pre-2025 $4,500 exemption, and counted that exemption twice as both a standard deduction and a personal exemption ($9,000 total). Under the reform the only offset is the 2026 indexed $12,835 standard deduction, which gives $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_later_law,taxable_income_or_deductions,False,"It used a superseded 4.25% rate instead of the flat 3%, and the old $4,500 exemption instead of the $12,835 standard deduction. It also cut AGI to $34,791 by subtracting the $5,389 employer insurance premiums from the given gross wages, when Louisiana AGI is $40,180."
+us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_later_law,taxable_income_or_deductions,False,"It used the base $12,500 single standard deduction without the 2026 inflation indexing that raises it to $12,835. That gave $830.40 instead of 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_later_law,taxable_income_or_deductions,False,"It subtracted the unindexed $12,500 deduction instead of the 2026 indexed $12,835, so taxable income came out at $27,680 rather than $27,345. That gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_later_law,taxable_income_or_deductions,False,"It applied the repealed pre-2025 regime: 1.85%/3.5% brackets, the $4,500 combined exemption, and a deduction for federal income tax paid. It also started from a $34,791 AGI that wrongly subtracts the ESI premiums. For 2026 the rule is a flat 3% on $40,180 minus the $12,835 standard deduction, which is $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_later_law,taxable_income_or_deductions,False,"It applied 'progressive' rates, but Louisiana has a single flat 3% rate in 2026. The flat 3% on $40,180 minus the $12,835 standard deduction gives $820.35, not $724."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_later_law,taxable_income_or_deductions,False,"It used the repealed 1.85%/3.5% brackets and the old $4,500 exemption. It also started from a $34,791 AGI that wrongly subtracts the employer insurance premiums. The 2026 law applies 3% to $40,180 minus the $12,835 standard deduction, which is $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_later_law,taxable_income_or_deductions,False,"It deducted federal income tax paid and used the $4,500 exemption and 1.85%/3.5% brackets, all of which the 2025 reform repealed. The 2026 computation is 3% × ($40,180 − $12,835) = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_later_law,taxable_income_or_deductions,False,"It gave no derivation. At 3%, $1,115 implies about $37,000 of taxable income, so it applied far less than the 2026 $12,835 standard deduction or used the repealed graduated rates. The correct figure is 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_later_law,taxable_income_or_deductions,False,"It used the pre-reform $4,500 combined exemption and 1.85%/3.5% brackets on a $34,791 AGI that wrongly excludes the ESI premiums. The 2026 law applies a flat 3% to $40,180 minus the $12,835 standard deduction, giving $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_later_law,taxable_income_or_deductions,False,"It applied the repealed $4,500 exemption and the 1.85%/3.5% brackets instead of the flat 3% rate and the $12,835 2026 standard deduction. That overstated taxable income by $8,335 and gave $1,043 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_later_law,taxable_income_or_deductions,False,"It used the pre-2025 $4,500 deduction and the 1.85%/3.5% graduated rates. Louisiana's 2026 law is a flat 3% after a $12,835 single standard deduction, which gives $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-4-argon,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction without the 2026 inflation indexing to $12,835. That gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,reference_later_law,taxable_income_or_deductions,False,"It treated the 2026 single standard deduction as $12,500, but it is indexed for inflation from 2026 to $12,835. Taxable income is therefore $27,345 and the tax is $820.35, not $830.40."
+us,scenario_051,state_income_tax_before_refundable_credits,glm-5.3,reference_later_law,taxable_income_or_deductions,False,"It applied the original $12,500 deduction from the 2025 reform without the inflation indexing that starts in 2026. The indexed $12,835 deduction gives 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_later_law,taxable_income_or_deductions,False,"It claimed the standard deduction fully offsets the tax, but $40,180 minus the $12,835 single standard deduction leaves $27,345 taxable. At the 3% flat rate that is $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_later_law,taxable_income_or_deductions,False,"It gave no computation. At 3%, $781 implies about $26,033 of taxable income, which means it used a deduction of roughly $14,150 instead of the $12,835 Louisiana standard deduction. The correct figure is 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 single standard deduction instead of the 2026 inflation-adjusted $12,835. That overstated taxable income by $335 and gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_later_law,taxable_income_or_deductions,False,"It subtracted $12,500, but the 2026 Louisiana single standard deduction is indexed to $12,835. That gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_later_law,taxable_income_or_deductions,False,"It used the base $12,500 deduction and left out the 2026 inflation indexing to $12,835. Taxable income is $27,345, so the tax is $820.35, not $830.40."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_later_law,taxable_income_or_deductions,False,"It applied the unindexed $12,500 single standard deduction. The 2026 indexed amount of $12,835 gives 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-astra,reference_later_law,taxable_income_or_deductions,False,"It used $12,500 as the single standard deduction instead of the 2026 inflation-indexed $12,835. That overstated the flat 3% tax by $10.05."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-luna,reference_later_law,taxable_income_or_deductions,False,"It estimated the 2026 single standard deduction at the unindexed $12,500, but Louisiana indexes it to $12,835 for 2026. Taxable income is $27,345 and the tax is $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-6-sol,reference_later_law,taxable_income_or_deductions,False,"It left out the 2026 inflation indexing of the $12,500 standard deduction to $12,835. That gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_later_law,taxable_income_or_deductions,False,"It subtracted the base $12,500 deduction instead of the 2026 indexed $12,835. That left $27,680 taxable instead of $27,345, and $830.40 of tax instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-6.1-terra,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction without the inflation adjustment to $12,835 that applies starting in 2026. That gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,reference_later_law,taxable_income_or_deductions,False,"It gave no derivation. At 3%, $680 implies about $22,667 of taxable income, which means it over-deducted by roughly $4,700 beyond the $12,835 standard deduction or used the repealed bracket structure. The correct figure is 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,reference_later_law,taxable_income_or_deductions,False,"It subtracted the federal $15,000 standard deduction instead of Louisiana's own $12,835 deduction. It also applied the repealed 1.85%/3.5% brackets instead of the flat 3% rate, giving $675 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-4.6,reference_later_law,taxable_income_or_deductions,False,"It used the unindexed $12,500 deduction (giving $830.40, rounded to $830) instead of the 2026 indexed $12,835. That deduction yields 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-4.7,reference_later_law,taxable_income_or_deductions,False,"It applied the base $12,500 single standard deduction and left out the 2026 inflation indexing to $12,835. That gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,reference_later_law,taxable_income_or_deductions,False,"It used the repealed pre-2025 regime: a deduction for federal tax, a $4,500 standard deduction plus a $4,500 personal exemption, and 1.85%/3.5% brackets. It also started from a $34,791 AGI that wrongly subtracts the ESI premiums. The 2026 law is 3% of ($40,180 − $12,835) = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,inkling,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction instead of the 2026 inflation-indexed $12,835 and rounded $830.40 to $830. The correct figure is 3% × $27,345 = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was no answer to score against the $820.35 reference."
+us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,reference_later_law,taxable_income_or_deductions,False,"It subtracted the unindexed $12,500 single standard deduction instead of the 2026 indexed $12,835. That left $27,680 taxable instead of $27,345, and $830.40 of tax instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,reference_later_law,taxable_income_or_deductions,False,"It used the repealed $4,500 standard deduction and left the capital gains out of income, computing $1,065. It then submitted $1,208, which its own work does not support. The correct figure is 3% × ($40,180 − $12,835) = $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,muse-spark-1.3,reference_later_law,taxable_income_or_deductions,False,"It used the base $12,500 deduction rather than the 2026 inflation-indexed $12,835. That gave $830.40 instead of $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,muse-wonder,reference_later_law,taxable_income_or_deductions,False,"It applied the unindexed $12,500 single standard deduction, but Louisiana's 2026 deduction is $12,835. Taxable income is $27,345 and the tax is $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,ox-alpha,reference_later_law,taxable_income_or_deductions,False,"It treated the 2026 standard deduction as $12,500 and left out the inflation indexing to $12,835. That overstated the tax by $10.05 ($830.40 instead of $820.35)."
+us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_later_law,taxable_income_or_deductions,False,"It subtracted the federal $15,750 standard deduction plus a repealed $4,500 exemption and applied made-up 1.75%/3.55% brackets, computing $482.52. It then submitted an unrelated $573. Louisiana's 2026 law is a flat 3% on $40,180 minus its own $12,835 deduction, which is $820.35."
+us,scenario_051,state_income_tax_before_refundable_credits,qwen3.8-max,reference_later_law,taxable_income_or_deductions,False,"Its $1,324.80 is more than 3% of the entire $40,180 AGI ($1,205.40). That means it used a rate above Louisiana's flat 3% and/or skipped the $12,835 standard deduction. The correct figure is 3% × $27,345 = $820.35."
us,scenario_051,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It used a $32,600 standard deduction instead of $32,200, and its bracket math gave about $106,932 of ordinary tax, roughly $3,000 too high (the 2026 MFJ brackets give about $103,866 on about $471,700). It then submitted $117,285, which contradicts its own computed total of $107,277."
us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It disallowed the $43 traditional IRA contribution as phased out for an active 401(k) participant, while the reference deducts $43.28 above the line (AGI $504,778.56 vs. the model's $504,822). Every other step matches: the $32,200 standard deduction, $103,879.68 on $471,774, $127.20 of QD tax and $218.42 of NIIT. That leaves a $13.89 gap."
@@ -4842,44 +4899,45 @@ us,scenario_052,payroll_tax,minimax-m3,llm_error,other,False,"The model computed
us,scenario_052,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's reasoning totals $20,418.20, using the outdated $176,100 wage base instead of $184,500. It then submitted $32,846, which none of its stated components support."
us,scenario_052,payroll_tax,qwen3.8-max,llm_error,other,False,"The model counted the $2,250 Additional Medicare Tax twice: it added it again to a $20,418.20 subtotal that already included it. It also used the outdated $176,100 Social Security wage base instead of the 2026 base of $184,500."
us,scenario_052,spouse_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified rule that makes a 55-year-old Medicare eligible. The spouse is below the age-65 threshold and has no listed disability or other qualifying Medicare pathway, yielding spouse_medicare_eligible = 0."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 of ESI premiums from wages and also took a $950 auto loan interest deduction. In this computation AGI is the full $66,969 and the auto loan interest is not deductible. Its own work reached $5,336, yet it submitted $6,224, a figure that matches no derivation; the correct result is 22% bracket tax on $50,869 of taxable income, or $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It took a $950 OBBBA auto loan interest deduction as an above-the-line adjustment. That deduction requires a qualifying new, US-assembled vehicle, and no such vehicle is among the listed facts, so it does not apply. Without it, taxable income is $50,869, which reaches into the 22% bracket above $50,400 and gives $5,903 instead of $5,742."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It subtracted the ESI premiums from wages, used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100, and applied made-up bracket arithmetic that came to $12,761.60. It then submitted $5,744, which is consistent with none of its own steps. The correct calculation is $66,969 − $16,100 = $50,869, taxed at 10%, 12% and 22%, for $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $3,389 of ESI premiums, which made taxable income $47,480 instead of $50,869. It then also understated the tax on that base: $4,700 instead of the $5,449.60 the 2026 brackets give. The correct AGI is the full wages, giving $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $16,600 standard deduction instead of the 2026 amount of $16,100 and treated taxable income as fitting entirely within the 12% bracket, getting $5,796. It then raised the answer to $6,196 with no supporting calculation. Correct taxable income is $50,869, with $468.68 taxed at 22%, for $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its own reasoning computed the correct $5,903 from $50,869 of taxable income and the 2026 brackets of $12,400 and $50,400. It then dropped that result and submitted $6,027 based on unsupported 'rounding to bracket estimates'."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly derived $5,903 from $50,869 of taxable income ($1,240 + $4,560 + $103). It then submitted $6,379, a number its reasoning does not support."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 bracket thresholds of $11,925 and $48,475 and an estimated $15,700 standard deduction. The 2026 OBBBA values are $12,400 and $50,400 for the brackets and $16,100 for the deduction. That pushed $2,794 into the 22% bracket instead of $468.68, giving $6,193 instead of $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It got the $16,100 deduction and $50,869 of taxable income right. However, it estimated the brackets as 2025 thresholds × 1.02 instead of using the 2026 thresholds of $12,400 and $50,400, and its loose arithmetic produced $6,154. The 2026 brackets give $1,240 + $4,560 + $103.11 = $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,650 standard deduction and bracket thresholds of $12,200 and $49,950 instead of the 2026 values of $16,100, $12,400 and $50,400. That made taxable income $51,319 and put $1,369 in the 22% bracket, giving $6,051.18 instead of $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the ESI premiums from wages to get AGI of $63,580 instead of $66,969 and used a $15,400 standard deduction instead of $16,100. It also used a $12,225 top for the 10% bracket instead of $12,400. Together these understated taxable income at $48,180 and left out the 22% portion, giving $5,537.10."
-us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expired in 2026 and applied an $8,300 standard deduction, a $5,300 personal exemption and 15%/25% brackets. OBBBA made the TCJA structure permanent: a $16,100 standard deduction, no personal exemption, and 10%/12%/22% brackets, which give $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the TCJA-expiration parameters: an $8,300 standard deduction, a $5,300 exemption, and 15%/25% brackets. It also subtracted the $3,389 of ESI premiums from wages. Under the 2026 OBBBA law, AGI is $66,969, the standard deduction is $16,100, and the 10%/12%/22% brackets give $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of $16,100, and its $3,939 is far below what any 2026 bracket calculation gives. Taxing $50,869 at 10% up to $12,400, 12% up to $50,400 and 22% above that comes to $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using pre-TCJA brackets plus a personal exemption, and subtracted the ESI premiums to get AGI of $63,580. 2026 law keeps the $16,100 standard deduction with no exemption and the 10%/12%/22% brackets on AGI of $66,969, which gives $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied TCJA-expiration parameters (an $8,300 standard deduction, a $5,300 exemption, and 15%/25% brackets) and subtracted the ESI premiums from wages. OBBBA made the $16,100 standard deduction and the 10%/12%/22% brackets permanent for 2026, and AGI is the full $66,969, which gives $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It submitted $3,737 with no stated calculation, a figure about $2,166 below the tax on $50,869 of taxable income. The correct result is $1,240 at 10% + $4,560 at 12% + $103.11 at 22% = $5,903.11. Its number reflects misapplied brackets or a greatly overstated deduction."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 of ESI premiums from wages, cutting taxable income to $47,480 instead of $50,869, and also miscalculated the tax on that base ($5,459.10 instead of $5,449.60). With AGI equal to the full $66,969 of wages, tax is $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It treated the $3,389 of ESI premiums as a pre-tax reduction, giving AGI of $63,580 and taxable income of $47,480, which is taxed entirely within the 12% bracket. AGI is the full $66,969 of wages, so taxable income is $50,869 and tax is $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the 2025 standard deduction of $15,000 and the 2025 bracket thresholds of $11,925 and $48,475. The 2026 values are $16,100, $12,400 and $50,400. That overstated taxable income at $51,969 and moved $3,494 into the 22% bracket, giving $6,347.18 instead of $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"Its $7,796.15 matches the TCJA-expiration calculation: a roughly $8,300 standard deduction plus a $5,300 exemption, with 15%/25% brackets. Under 2026 OBBBA law, a $16,100 standard deduction and the 10%/12%/22% brackets give $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used 2024 parameters: a $14,600 standard deduction and bracket thresholds of $11,600 and $47,150. The 2026 values are $16,100, $12,400 and $50,400. That made taxable income $52,369 with $5,219 taxed at 22%, giving $6,574.18 instead of $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It reduced the tax by a 'basic CTC' and other nonrefundable credits, but this single filer has no children or other qualifying dependents, so no nonrefundable credits apply. Tax before refundable credits is the full $5,903 on $50,869 of taxable income, not $3,657."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It submitted $5,118 using 'approximated' 2026 parameters, which is less than even the $5,800 owed on the first $50,400 under the 2026 10%/12% brackets. The correct calculation is $66,969 − $16,100 = $50,869, giving $1,240 + $4,560 + $103.11 = $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a $16,000 standard deduction instead of $16,100. It also misapplied the brackets: $5,793 is below the $5,800 owed on the first $50,400 alone, so it never taxed the income above $50,400 at 22%. The correct total on $50,869 is $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It got the $16,100 deduction and $50,869 of taxable income right, but applied the 22% rate starting below $50,400. Its $5,985 implies the 12% bracket topped out near $50,030. Under the actual 2026 thresholds only $468.68 is taxed at 22%, for $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted $3,389 of ESI premiums as pretax, cutting taxable income to $47,480. In this computation AGI is the full $66,969, so taxable income is $50,869 and $468.68 is taxed at 22%, giving $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a standard deduction of about $15,000 (the 2025 level) instead of the 2026 amount of $16,100 and gave a rough bracket estimate of $5,850, which is not tied to the 2026 thresholds. The correct calculation is $50,869 of taxable income taxed at 10% up to $12,400, 12% up to $50,400 and 22% above, for $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA expired, applying an $8,300 standard deduction, a $5,300 exemption and 15%/25% brackets. 2026 law under OBBBA keeps the $16,100 standard deduction and the 10%/12%/22% brackets, which give $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied inflation-adjusted pre-TCJA parameters: an $8,522 standard deduction, a $5,435 exemption and 15%/25% brackets. OBBBA made the TCJA structure permanent, so the $16,100 standard deduction and 10%/12%/22% brackets apply, giving $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $16,150 standard deduction and bracket thresholds of $12,225 and about $49,725. The 2026 values are $16,100, $12,400 and $50,400. That put $1,094 into the 22% bracket instead of $468.68, giving $5,963 instead of $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied a pre-TCJA standard deduction of $8,372 with 15%/25% brackets, giving taxable income of $58,597 and tax of $9,073. 2026 law uses the $16,100 standard deduction and 10%/12%/22% brackets, which give $50,869 of taxable income and $5,903 of tax."
-us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It assumed the TCJA expired, using an $8,300 deduction, a $5,300 exemption and 15%/25% brackets, and subtracted the ESI premiums to get AGI of $63,580. 2026 OBBBA law applies the $16,100 standard deduction and 10%/12%/22% brackets to the full $66,969, giving $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used the outdated $13,850 standard deduction instead of $16,100, which overstated taxable income at $53,119 and produced $6,397.98 in its reasoning. It then submitted an unrelated $4,272. The 2026 deduction gives $50,869 of taxable income and $5,903 of tax."
-us,scenario_053,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used the 2025 standard deduction of $15,750 instead of the 2026 amount of $16,100, which raised taxable income to $51,219, and its bracket total of $6,035 is approximate. The correct taxable income of $50,869 gives $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It got the $16,100 deduction and $50,869 of taxable income right, but set the top of the 12% bracket at $50,150 instead of $50,400. Its $4,530 of 12% tax and 22% applied to $719 overtaxed $250 by 10 points, giving $5,928.18 instead of $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the 2025 bracket thresholds of $11,925 and $48,475 instead of $12,400 and $50,400, and a $16,000 deduction instead of $16,100, which gave $6,127 in its reasoning. It then submitted an unrelated $8,252. The correct 2026 calculation gives $5,903."
-us,scenario_053,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It reduced the tax by a child tax credit, dependent credit and other nonrefundable credits that this childless single filer cannot claim, and it understated taxable income at $49,779. The correct figures are $50,869 of taxable income with no credits, for $5,903."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated the $3,389 ESI premiums as pre-tax, cutting AGI to $63,580, and also deducted the $950 auto-loan interest. Neither applies: the facts give gross wages and never establish salary-reduction treatment or a qualifying new US-assembled vehicle loan. It then submitted $6,224, which doesn't match its own $5,335.60 calculation. Correct taxable income is $50,868.68, and tax is $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It subtracted $950 of auto-loan interest under the OBBBA vehicle-loan-interest deduction. That deduction requires a new vehicle with final assembly in the US and a loan originated after 2024, and the facts don't establish any of that. This put taxable income at $49,919 instead of $50,868.68 and left out the $103.11 of 22%-bracket tax."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It took the $950 qualified passenger-vehicle loan interest deduction, but nothing in the facts shows the vehicle is a qualifying new, US-assembled vehicle. As a result its taxable income was $49,919 rather than $50,868.68, and it missed the $468.68 that falls in the 22% bracket above $50,400."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premiums from gross wages and used the 2024 standard deduction of $14,600 instead of the 2026 figure of $16,100. Its bracket math was garbled (it applied 22% from $11,000) and gave $12,761.60, and it then submitted $5,744, which doesn't match. The correct calculation is $50,868.68 of taxable income on the $12,400/$50,400 brackets, giving $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premiums, which the facts don't establish as a pre-tax exclusion, giving taxable income of $47,480 instead of $50,868.68. It also misapplied the brackets: even its own taxable income yields $5,449.60, not $4,700."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $16,600 standard deduction instead of the 2026 single amount of $16,100 and arrived at $5,796. It then raised that to $6,196 with no stated basis. The correct taxable income of $50,868.68 puts $468.68 in the 22% bracket, for $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its own derivation ($16,100 deduction, $50,869 taxable income, 10%/12%/22% brackets at $12,400/$50,400) produced the correct $5,903. It then threw that away and submitted $6,027 based on an unexplained ""rounding to bracket estimates."""
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly worked out $1,240 + $4,560 + $103 ≈ $5,903 on $50,869 of taxable income. It then replaced that with an unsupported $6,379, so the submitted value contradicts its own correct calculation."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It guessed the 2026 standard deduction at $15,700 and used 2025 brackets (10% to $11,925, 12% to $48,475). The correct 2026 OBBBA values are a $16,100 deduction and brackets at $12,400 and $50,400. Its numbers pushed $2,794 into the 22% bracket instead of $468.68 and overstated the tax at $6,193."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It had the correct $16,100 deduction and $50,869 taxable income. But it built the brackets by inflating 2025 thresholds by 2%, when the actual 2026 brackets are 10% to $12,400 and 12% to $50,400. This put too much income in the 22% bracket and gave $6,154 instead of $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,650 standard deduction and brackets of 10% to $12,200 and 12% to $49,950. The 2026 values are $16,100, $12,400, and $50,400. This inflated taxable income to $51,319 and put $1,369 in the 22% bracket, giving $6,051.18."
+us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premiums from wages, which the facts don't support, and used a $15,400 standard deduction instead of $16,100. That gave taxable income of $48,180 rather than $50,868.68, and with a $12,225 10%-bracket cap the tax came out at $5,537.10."
+us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA law: an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. The OBBBA made the TCJA structure permanent, so 2026 has a $16,100 deduction, no personal exemption, and 10%/12%/22% brackets at $12,400 and $50,400."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and used an $8,300 deduction, a $5,300 exemption, and 15%/25% rates. The OBBBA kept the $16,100 deduction and the 10%/12%/22% schedule for 2026. It also subtracted the $3,389 ESI premiums from wages, though the facts don't establish them as a pre-tax exclusion."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of $16,100. Its $3,939 is also far below what any correct bracket application gives: its own $51,669 taxable income produces about $6,079 under the 2026 schedule. The correct figure is $50,868.68 taxable income and $5,903.11 tax."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA expired and used pre-TCJA deduction, exemption, and 15%/25% bracket rules, but the OBBBA kept the 2026 $16,100 deduction and the 10%/12%/22% brackets. It also subtracted the $3,389 ESI premiums, cutting AGI to $63,580 instead of using full wages of $66,968.68."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used TCJA-sunset parameters: an $8,300 deduction, a $5,300 exemption, and 15%/25% brackets. The OBBBA keeps a $16,100 deduction with the 10%/12%/22% brackets at $12,400 and $50,400. It also wrongly subtracted the $3,389 ESI premiums from AGI."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no computation. The correct path is $66,968.68 − $16,100 = $50,868.68 taxable, taxed at 10%/12%/22% with thresholds of $12,400 and $50,400, giving $5,903.11. Its $3,737 matches only about $33,200 of taxable income under that schedule, so it either overstated deductions or misapplied the brackets."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $3,389 ESI premiums from gross wages, which the facts don't support, so taxable income came out at $47,480 instead of $50,868.68. That dropped the 12%-bracket income above $47,480 and the 22%-bracket income above $50,400."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $63,580 by treating the $3,389 ESI premiums as a pre-tax exclusion, which the facts don't support. That gave $47,480 of taxable income and $5,449.60 of tax, instead of $50,868.68 and $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the original 2025 standard deduction of $15,000 and 2025 brackets (10% to $11,925, 12% to $48,475). The 2026 figures are a $16,100 deduction and thresholds of $12,400 and $50,400. That overstated taxable income by $1,100 and put $3,494 in the 22% bracket."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"It gave no computation, but its $7,796.15 matches a pre-TCJA calculation (an $8,300 deduction plus a personal exemption, with 15%/25% brackets). The OBBBA keeps the 2026 $16,100 deduction and the 10%/12%/22% schedule, which gives $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used 2024 parameters: a $14,600 standard deduction and brackets of 10% to $11,600 and 12% to $47,150. The 2026 values are $16,100, $12,400, and $50,400. This overstated taxable income at $52,369 and put $5,219 in the 22% bracket."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It subtracted a ""basic CTC"" and other nonrefundable credits even though the head is a single 25-year-old with no qualifying children or credit-generating expenses. No nonrefundable credit applies, so the tax is the full $5,903.11 on $50,868.68 of taxable income."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It relied on approximate brackets and deduction values instead of the 2026 $16,100 deduction and 10%/12%/22% thresholds at $12,400 and $50,400. Its $5,118 matches only about $44,350 of taxable income, roughly $6,500 less than the correct $50,868.68."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a $16,000 deduction instead of $16,100 and then misapplied the brackets. Its own $50,969 taxable income yields $5,925 under the 2026 10%/12%/22% thresholds of $12,400 and $50,400, not $5,793."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It had the correct $50,869 taxable income but misapplied the brackets. The 12% bracket ends at $50,400, so only $468.68 is taxed at 22% ($103.11), giving $1,240 + $4,560 + $103.11 = $5,903.11 rather than $5,985."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted $3,389 of ""pretax employer-plan premiums"" from wages, but the facts give gross wages and don't establish a salary-reduction exclusion. Full wages of $66,968.68 are AGI, so taxable income is $50,868.68, not $47,480."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a standard deduction of about $15,000 instead of the 2026 single amount of $16,100, and its bracket math was rough. The correct taxable income is $50,868.68, and the $12,400/$50,400 thresholds give $5,903.11, not $5,850."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: an $8,300 deduction, a $5,300 personal exemption, and 15%/25% rates. The OBBBA made the $16,100 deduction and the 10%/12%/22% brackets permanent, so 2026 taxable income is $50,868.68 and tax is $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and used an $8,522 deduction, a $5,435 personal exemption, and 15%/25% brackets. The 2026 rules under the OBBBA are a $16,100 deduction, no exemption, and 10%/12%/22% brackets at $12,400 and $50,400."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used a $16,150 deduction and brackets of 10% to $12,225 and 12% to $49,725. The 2026 figures are $16,100, $12,400, and $50,400. This put $1,094 in the 22% bracket instead of $468.68 and gave $5,963."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA law with an $8,372 standard deduction and 15%/25% brackets. The OBBBA keeps a $16,100 deduction and the 10%/12%/22% schedule in 2026, so taxable income is $50,868.68, not $58,597."
+us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It assumed the TCJA expired and used an $8,300 deduction, a $5,300 exemption, and 15%/25% brackets, but the OBBBA keeps the $16,100 deduction and the 10%/12%/22% brackets in 2026. It also wrongly subtracted the $3,389 ESI premiums from AGI."
+us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used the 2023 standard deduction of $13,850 instead of the 2026 amount of $16,100, which overstated taxable income at $53,119. It then submitted $4,272, which doesn't match its own $6,397.98 calculation."
+us,scenario_053,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used the 2025 standard deduction of $15,750 instead of the 2026 amount of $16,100, which inflated taxable income to $51,219. Its $6,035 is also above what that income yields on the $12,400/$50,400 brackets (about $5,980)."
+us,scenario_053,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It had the correct $50,869 taxable income but ended the 12% bracket at $50,150 instead of $50,400. That gave $4,530 at 12% instead of $4,560 and taxed $719 at 22% instead of $468.68."
+us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $16,000 deduction and the 2025 brackets (10% to $11,925, 12% to $48,475) instead of the 2026 values of $16,100, $12,400, and $50,400, and its calculation gave $6,127. It then submitted $8,252, which doesn't match that calculation."
+us,scenario_053,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It reduced the tax by a nonrefundable child tax credit and other dependent credits even though the household is a single adult with no dependents. It also understated taxable income at $49,779 instead of $66,968.68 − $16,100 = $50,868.68."
us,scenario_053,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a refundable child or other-dependent credit despite the household containing no qualifying child or dependent, and it also treated this high-income childless filer as receiving refundable EITC. Neither eligibility pathway applies, so the claimed $1,966 credit has no qualifying component."
us,scenario_053,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model applied the correct 6.2% Social Security and 1.45% Medicare rates to a base of $63,580. That base came from subtracting the $3,389 employer-sponsored insurance premium, which treats the premium as a pre-tax Section 125 exclusion the household facts never state. FICA applies to the full $66,969 of gross wages, and 7.65% of that is $5,123.10."
us,scenario_053,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model explicitly deducted the $3,389 employer-sponsored insurance premium as a pre-tax payroll exclusion ($66,969 - $3,389 = $63,580) before applying 7.65%. The payroll-tax wage base is the full $66,969 of gross wages, so the correct total is $4,152.06 Social Security plus $971.05 Medicare, or $5,123.10."
@@ -4892,57 +4950,58 @@ us,scenario_053,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model set up
us,scenario_053,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model applied 7.65% to $63,580 'after pretax employer-plan premiums'. That subtracts the $3,389 employer-sponsored insurance premium as a cafeteria-plan exclusion the household facts do not state. FICA is levied on the full $66,969 of gross wages, which gives $5,123.10."
us,scenario_053,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model defined taxable payroll wages as gross wages minus the $3,389 employer-sponsored insurance premium ($63,580), then applied 6.2% and 1.45% and rounded to $4,864. The premium is not excluded from the payroll-tax base. On the full $66,969 of wages, Social Security is $4,152.06 and Medicare is $971.05, for $5,123.10 in total."
us,scenario_053,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to an invented $53,000 base instead of 5.3%. It never took the $5,000 Idaho health insurance premium subtraction, the $4,811 zero-rate bracket, or the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It started from a wrong AGI of $66,019 and applied 5.3% to all $49,919. It skipped the $4,811 zero-rate bracket and the $5,000 Idaho health insurance premium subtraction, and it left out the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It used Idaho's old progressive brackets (up to 7.15%) and a $7,200 standard deduction instead of the 2026 flat 5.3% rate above $4,811 with the $16,100 standard deduction. It subtracted the ESI premiums rather than taking the $5,000 Idaho health premium subtraction, and it omitted the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It gave a rounded $2,400 guess at 5.3% without working out the base. The correct base is $66,968.68 minus the $5,000 health premium subtraction, the $16,100 standard deduction and the $4,811 zero bracket, which gives $2,176.06; the $10 building fund tax then brings it to $2,186.06."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It used a $15,750 standard deduction instead of $16,100 and moved between 5.3%, 5.695% and 4.7% before guessing $2,548. It never applied the $4,811 zero-rate bracket, the $5,000 Idaho health insurance premium subtraction, or the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It used the repealed 5.695% rate and a $4,673 threshold instead of 5.3% above $4,811. Its submitted $2,867 does not match its own $2,631 arithmetic. It also missed the $5,000 health premium subtraction and the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It skipped the $5,000 Idaho health insurance premium subtraction and treated the refundable grocery credit as a nonrefundable offset. Its submitted $2,650 does not follow from its own 5.3% calculation above $4,900, and it omitted the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Its rate (5.3%) and zero bracket ($4,811) were correct, but it never subtracted the $5,000 of health insurance premiums the household pays, which Idaho allows. That overstated tax by $265. It also left out the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It wrongly reduced federal AGI by the $3,389 ESI premiums and used a $15,000 standard deduction and the repealed 5.695% rate. It never applied the $4,811 zero bracket, the $5,000 Idaho health premium subtraction, or the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It used the repealed 5.695% rate and a $15,000 standard deduction, and its final $3,706 does not even match its own $2,960. It said health premiums are not deductible for Idaho, which misses the $5,000 subtraction. It also missed the $4,811 zero bracket and the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It applied 5.3% to the full $50,869 and skipped the $4,811 zero-rate bracket. It also missed the $5,000 Idaho health insurance premium subtraction and the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It made up a $4,000 personal exemption and a 5.59% rate, and it used a $15,650 standard deduction. The actual structure is 5.3% above a $4,811 zero bracket after the $16,100 deduction and the $5,000 health premium subtraction. It also left out the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It applied 5.3% to an unexplained $48,180 base with no $4,811 zero-rate bracket. It also omitted the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It used an $8,300 standard deduction instead of $16,100 and taxed the whole $58,669 at 5.3%. It skipped the $4,811 zero bracket, the $5,000 health premium subtraction, and the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 and skipped the $4,811 zero-rate bracket. It also missed the $5,000 Idaho health insurance premium subtraction and the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used the repealed 5.695% rate instead of 5.3% and subtracted a nonexistent personal exemption. It never applied the $4,811 zero bracket, the $5,000 health premium subtraction, or the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave an unsupported $2,652, which matches a flat rate on AGI minus only the standard deduction. The correct method takes 5.3% of ($66,969 − $5,000 health premium subtraction − $16,100 − $4,811 zero bracket) and adds the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It applied a made-up 5.692% rate to roughly $49,852, with no $4,811 zero bracket and no $5,000 Idaho health premium subtraction. It also omitted the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It reduced federal AGI by the ESI premiums and used the repealed 5.8% rate instead of 5.3% above $4,811. It also subtracted a nonexistent personal exemption, missed the $5,000 health premium subtraction, and left out the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave an unsupported $3,450, which matches an outdated higher rate on a base with no zero bracket or health premium subtraction. It never computed 5.3% of $41,058 plus the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It subtracted the $3,389 ESI premiums from AGI instead of the $5,000 Idaho subtraction for health premiums the household pays. It used the repealed 5.695% rate with no $4,811 zero bracket and omitted the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to roughly $47,470 (AGI minus ESI premiums minus the standard deduction). It missed the 5.3% rate, the $4,811 zero bracket, the $5,000 health premium subtraction, and the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It used a $15,000 standard deduction and the repealed 5.695% rate. It never applied the $4,811 zero bracket, the $5,000 health premium subtraction, or the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"It took the standard deduction and the health premium subtraction, but its $2,401.29 matches taxing about $45,300 at 5.3% instead of only the $41,058 above the $4,811 zero-rate bracket. It also omitted the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It invented a deduction of $1,250 per $1,000 of income, which wiped out taxable income. In fact Idaho taxable income is $45,869 after the $16,100 standard deduction and $5,000 health premium subtraction, taxed at 5.3% above $4,811, plus the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It used a made-up 5.6% rate on the full $50,869, with no $4,811 zero bracket. It also missed the $5,000 Idaho health premium subtraction and the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It used nonexistent progressive 'state brackets' instead of the flat 5.3% rate above $4,811. Its $2,050 understates the correct 5.3% × $41,057.68 = $2,176.06, and it left out the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It gave an unsupported $1,195, about half the liability. The correct figure is 5.3% of ($66,968.68 − $5,000 − $16,100 − $4,811) = $2,176.06 plus the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It used a $15,750 standard deduction instead of $16,100 and a $4,489 zero bracket instead of $4,811. It also never took the $5,000 health premium subtraction or added the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 and skipped the $4,811 zero-rate bracket. It also missed the $5,000 Idaho health insurance premium subtraction and the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It used a $4,946 zero bracket instead of $4,811 and never took the $5,000 Idaho subtraction for health insurance premiums, so it taxed $45,923 instead of $41,058. It also omitted the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 and skipped the $4,811 zero-rate bracket. It also missed the $5,000 health premium subtraction and the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It correctly computed the $2,176 income tax using 5.3%, the $16,100 deduction, the $5,000 health premium subtraction and the $4,811 zero bracket. It left out the $10 permanent building fund tax, which is part of tax before refundable credits."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It used repealed progressive brackets on $50,869 instead of the flat 5.3% above $4,811. It also missed the $5,000 health premium subtraction, so its result lands near the right figure by coincidence. It omitted the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Its 5.3% rate was right, but it never subtracted the $5,000 of health insurance premiums the household pays, so it taxed about $46,069 instead of $41,058. It also omitted the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It subtracted the $3,389 ESI premiums from federal AGI instead of taking the $5,000 Idaho subtraction for health premiums the household pays, which leaves taxable income $1,611 too high. It also omitted the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 and skipped the $4,811 zero-rate bracket. It also missed the $5,000 health premium subtraction and the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It used the repealed 5.8% rate on about $52,000 instead of 5.3% above $4,811. It also missed the $5,000 health premium subtraction and the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It used an $8,300 standard deduction instead of $16,100 and the repealed 5.695% rate. It also skipped the $4,811 zero bracket, the $5,000 health premium subtraction, and the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It used an $8,522 standard deduction instead of $16,100 and the repealed 5.695% rate. It also skipped the $4,811 zero bracket, the $5,000 health premium subtraction, and the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It used the 5.3% rate and a zero bracket, but it never subtracted the $5,000 of health insurance premiums the household pays, which Idaho allows. It also left out the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It applied the repealed 5.8% rate to $58,597, which implies a standard deduction far below $16,100. It also skipped the $4,811 zero bracket, the $5,000 health premium subtraction, and the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 and skipped the $4,811 zero-rate bracket. It also missed the $5,000 Idaho health insurance premium subtraction and the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for this variable, so there is no answer to score."
-us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 and skipped the $4,811 zero-rate bracket. It also missed the $5,000 health premium subtraction and the $10 permanent building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It said no standard deduction applies and taxed full AGI at the repealed 5.695% rate, then submitted $2,878, which does not match its own $3,814. It skipped the $16,100 standard deduction, the $5,000 health premium subtraction, the $4,811 zero bracket, and the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to $51,219, using a $15,750 deduction instead of $16,100. It never applied the $4,811 zero bracket, the $5,000 health premium subtraction, or the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"It applied the repealed 5.695% rate to all $50,869 instead of 5.3% above $4,811. It also missed the $5,000 health premium subtraction and the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It treated the $10 permanent building fund as a credit and subtracted it, when it is a tax that gets added. It also applied 5.3% to all $50,869, skipping the $4,811 zero bracket and the $5,000 Idaho health insurance premium subtraction."
-us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It used the repealed 5.8% rate and subtracted the ESI premiums instead of taking the $5,000 Idaho health premium subtraction. It reached $2,702 by an unexplained deduction step, without the $4,811 zero bracket or the $10 building fund tax."
-us,scenario_053,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It gave an unsupported $1,588 with no computation, which understates the liability. The correct figure is 5.3% of ($66,968.68 − $5,000 − $16,100 − $4,811) = $2,176.06 plus the $10 building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used the old 5.695% rate on a made-up $53,000 base instead of the 2026 5.3% rate on income above the $4,811 zero bracket. It never applied Idaho's $5,000 health insurance premium subtraction and left out the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It applied 5.3% to all $49,919 of taxable income, built on a made-up AGI of $66,019. It skipped the $4,811 zero-rate amount, never took Idaho's $5,000 health insurance premium subtraction, and left out the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It got the 5.3% rate, the $4,811 zero bracket, and the $10 permanent building fund tax right. It missed Idaho's subtraction for the $5,000 of health insurance premiums the household pays, so it taxed $50,869 instead of $45,868.68 and came out about $265 too high."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used Idaho's repealed graduated brackets, topping out at 7.15%, a $7,200 standard deduction, and a made-up $59 credit. Idaho actually applies a flat 5.3% above $4,811 after the $16,100 federal-conforming standard deduction. It also took out the ESI premium instead of the $5,000 Idaho health premium subtraction and left out the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It roughly applied 5.3% to taxable income after the standard deduction and stopped there. It skipped the $5,000 Idaho health insurance premium subtraction and the $4,811 zero-rate amount, and it did not add the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction instead of $16,100. It then switched between 5.3%, 5.695%, and 4.7% and picked $2,548 without applying the $4,811 zero bracket, the $5,000 Idaho health premium subtraction, or the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the outdated 5.695% rate with a $4,673 threshold instead of 5.3% above $4,811. Its submitted $2,867 is higher than even its own $2,631 calculation. It never took the $5,000 health premium subtraction or added the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It taxed $50,869 at 5.3% above about $4,900, which gives about $2,436, then raised the answer to $2,650 by wrongly treating the refundable grocery credit as relevant. It missed the $5,000 Idaho health insurance premium subtraction and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It got the 5.3% rate and the $4,811 zero bracket right, but it taxed $50,869 because it missed Idaho's $5,000 subtraction for health insurance premiums the household pays. It also did not add the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It took the $3,389 ESI premium out of AGI, which the reference does not do, used a $15,000 standard deduction instead of $16,100, and applied the old 5.695% rate with no $4,811 zero bracket. It counted the $5,000 of premiums only as an itemized medical expense, not as Idaho's separate health insurance subtraction, and it did not add the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It calculated $2,960 using 5.695% on $51,969, then submitted an unexplained $3,706. The correct method is 5.3% on Idaho taxable income above $4,811, after the $5,000 health premium subtraction and the $16,100 standard deduction, plus the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 of taxable income. It skipped the $4,811 zero-rate amount, the $5,000 Idaho health insurance premium subtraction, and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It made up a $4,000 personal exemption and a 5.59% rate and used a $15,650 standard deduction. Idaho has no personal exemption; it applies 5.3% above the $4,811 zero bracket after the $16,100 standard deduction and the $5,000 health premium subtraction, then adds the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It applied 5.3% to all of a $48,180 taxable income that doesn't match any correct deduction. It skipped the $4,811 zero bracket, did not apply the $5,000 health premium subtraction alongside the $16,100 standard deduction, and left out the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used an $8,300 standard deduction instead of the federal-conforming $16,100. It also skipped the $5,000 health premium subtraction and the $4,811 zero bracket and left out the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 of taxable income. It skipped the $4,811 zero-rate amount, the $5,000 Idaho health insurance premium subtraction, and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the outdated 5.695% rate to $49,980 with no zero bracket. The 2026 rate is 5.3% above $4,811, on taxable income reduced by the $5,000 Idaho health premium subtraction, plus the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Its $2,652 is roughly a flat rate applied to income after only the standard deduction. The correct result comes from subtracting the $5,000 of health premiums, applying 5.3% only above $4,811, and adding the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied a 5.692% rate to about $49,852 with no zero bracket instead of 5.3% above $4,811. It missed the $5,000 Idaho health premium subtraction and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It took the $3,389 ESI premium out of AGI, used the repealed 5.8% rate, and had no zero bracket. Idaho applies 5.3% above $4,811 after the $5,000 health premium subtraction and the $16,100 standard deduction, then adds the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $3,450 matches an old, higher Idaho rate applied with little or no deduction. The correct path is 5.3% on Idaho taxable income above $4,811, after the $5,000 health premium subtraction and the $16,100 standard deduction, plus the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It took the $3,389 ESI premium out of AGI instead of the $5,000 Idaho health premium subtraction and applied the outdated 5.695% rate to all $47,480. The correct rate is 5.3% above the $4,811 zero bracket, and it also left out the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied the outdated 5.695% flat rate to about $47,470, which matches taking out the ESI premium instead of the $5,000 Idaho health premium subtraction. It skipped the 5.3% rate, the $4,811 zero bracket, and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction instead of $16,100 and the outdated 5.695% rate on all $51,969. It missed the 5.3% rate above $4,811, the $5,000 health premium subtraction, and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"It applied the standard deduction and the health premium subtraction to reach about $45,869, but its $2,401.29 is close to taxing almost all of that income, as if the $4,811 zero-rate amount did not exist. It also left out the $10 permanent building fund tax. The correct result is ($45,868.68 − $4,811) × 5.3% + $10 = $2,186.06."
+us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It made up an Idaho deduction of $1,250 per $1,000 of income, which wiped out all taxable income. Idaho actually allows the $16,100 standard deduction and the $5,000 health premium subtraction, which leaves $45,868.68 taxable at 5.3% above $4,811, plus the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a made-up 5.6% flat rate on all $50,869 with no $4,811 zero bracket. It never took the $5,000 Idaho health premium subtraction or added the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It estimated $2,050 using vague 'state brackets' instead of working through the calculation. The correct result is ($61,968.68 − $16,100 − $4,811) × 5.3% = $2,176.06 plus the $10 permanent building fund tax, so its estimate is too low."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $1,195 implies an effective rate under 3% on Idaho taxable income, far below the correct 5.3% applied above $4,811 to $45,868.68. It also gave no sign of the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used the right 5.3% rate-plus-zero-bracket method but with stale figures: a $15,750 standard deduction instead of $16,100 and a $4,489 threshold instead of $4,811. It also missed the $5,000 Idaho health premium subtraction and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 of taxable income. It skipped the $4,811 zero-rate amount, the $5,000 Idaho health insurance premium subtraction, and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied 5.3% above a $4,946 zero bracket, close to the correct $4,811, but taxed $50,869 because it missed Idaho's $5,000 subtraction for health premiums the household pays. It also left out the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 of taxable income. It skipped the $4,811 zero-rate amount, the $5,000 Idaho health insurance premium subtraction, and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It got the Idaho income tax right at about $2,176, using the $5,000 health premium subtraction, the $16,100 standard deduction, the $4,811 zero bracket, and the 5.3% rate. It left out the $10 permanent building fund tax that Idaho adds to state income tax before refundable credits."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It applied progressive brackets that no longer exist to $50,869. Idaho applies a flat 5.3% above $4,811, and taxable income should first be reduced by the $5,000 health premium subtraction to $45,868.68. It also left out the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It applied 5.3% above roughly a $4,800 zero bracket but started from $50,869 because it missed Idaho's $5,000 health insurance premium subtraction. It also left out the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It reduced income by the $3,389 ESI premium, reaching $47,480, instead of by Idaho's $5,000 subtraction for premiums the household pays, which gives $45,868.68. It also left out the $10 permanent building fund tax, although its 5.3% rate and $4,811 threshold were correct."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 of taxable income. It skipped the $4,811 zero-rate amount, the $5,000 Idaho health insurance premium subtraction, and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used the repealed 5.8% rate on about $52,000 with no zero bracket. It never took the $5,000 Idaho health premium subtraction or added the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used an $8,300 standard deduction instead of $16,100 and the outdated 5.695% rate with no $4,811 zero bracket. It also missed the $5,000 health premium subtraction and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used an $8,522 standard deduction instead of $16,100 and the outdated 5.695% rate with no $4,811 zero bracket. It also missed the $5,000 health premium subtraction and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It used the 5.3% rate above a roughly $4,942 zero bracket but missed Idaho's $5,000 health insurance premium subtraction, so it taxed about $50,800 instead of $45,868.68. It also left out the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied the repealed 5.8% rate to $58,597, which implies a standard deduction of about $8,400 instead of $16,100. It had no $4,811 zero bracket, no $5,000 health premium subtraction, and no $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 of taxable income. It skipped the $4,811 zero-rate amount, the $5,000 Idaho health insurance premium subtraction, and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score against the $2,186.06 reference."
+us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It applied 5.3% to all $50,869 of taxable income. It skipped the $4,811 zero-rate amount, the $5,000 Idaho health insurance premium subtraction, and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Its reasoning applied 5.695% to all $66,969 of AGI with no standard deduction, then it submitted an unrelated $2,878. The correct path takes the $5,000 health premium subtraction and the $16,100 standard deduction, applies 5.3% above $4,811, and adds the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It applied the outdated 5.695% rate to $51,219, which uses a $15,750 standard deduction, with no $4,811 zero bracket. It missed the $5,000 health premium subtraction and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It applied the outdated 5.695% rate to all $50,869 instead of 5.3% above the $4,811 zero bracket. It missed the $5,000 Idaho health premium subtraction and the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It treated the $10 permanent building fund as a nonrefundable credit and subtracted it, when it is a tax that must be added. It also applied 5.3% to all $50,869, skipping the $4,811 zero bracket and the $5,000 Idaho health premium subtraction."
+us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used the repealed 5.8% rate, took the $3,389 ESI premium out of wages, and landed on $2,702 through vague deduction handwaving. The correct path is 5.3% above $4,811 on $45,868.68, after the $5,000 health premium subtraction and the $16,100 standard deduction, plus the $10 permanent building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It gave $1,588 with no shown calculation, which implies an effective rate well below Idaho's 5.3%. The correct result is ($61,968.68 − $16,100 − $4,811) × 5.3% = $2,176.06 plus the $10 permanent building fund tax."
us,scenario_053,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model wrongly said Idaho's grocery credit is treated as nonrefundable and that a $66,969 earner is over its income limit. The credit is a refundable state credit with no income test, worth $155 for one resident under 65 in 2026."
us,scenario_053,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model looked only for a state EITC and claimed Idaho has no refundable credits at this income. It missed the refundable grocery credit, which has no income limit and pays $155 per under-65 resident in 2026."
us,scenario_053,state_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,The model said no refundable state credit applies and missed Idaho's refundable grocery credit. This adult under 65 qualifies for all 12 months and gets $155 in 2026.
@@ -5087,55 +5146,56 @@ us,scenario_054,self_employment_tax,gpt-5.4-mini,llm_error,other,False,"The mode
us,scenario_054,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model computed $17,100 × 0.9235 × 0.153 ≈ $2,414, which is close to correct. It then claimed that excluding Additional Medicare Tax and rounding gave $1,596. Additional Medicare Tax is $0 at these earnings, and nothing should be subtracted from regular SECA, so the answer is $2,416.15."
us,scenario_054,self_employment_tax,minimax-m3,llm_error,other,False,"The model got net SE earnings of $15,792 and SE tax of about $2,416.18, which is essentially correct. It then submitted $2,419.95, a number that matches none of its own figures. The correct value is $15,791.85 × 0.153 = $2,416.15."
us,scenario_054,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model correctly reached $2,416 of SE tax. It then applied 0.9235 a second time and subtracted the deductible half of SE tax from the SE tax itself, producing $1,578. The half-SE-tax deduction is an above-the-line deduction against income tax and does not reduce SE tax, which stays at $2,416.15."
-us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,"The deduction steps were close to correct (net about $929), but the model projected the 3-person maximum allotment at about $820 instead of the $785 PolicyEngine uses. That inflated the monthly benefit to $540 instead of $505."
-us,scenario_054,snap,claude-fable-5.1,llm_error,other,False,"Max allotment ($785), standard deduction ($209), earned income deduction ($285) and net income ($931) were all correct. The model left the 30% contribution unrounded at $279.30 instead of rounding it up to $280, so it got $505.70/month rather than $505."
-us,scenario_054,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied a liquid-asset limit to the $14,000 bank balance. It missed that North Carolina's broad-based categorical eligibility through TANF non-cash benefits waives the SNAP asset test, and the household's gross income is only 64% of the poverty guideline."
-us,scenario_054,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model zeroed the benefit on asset and income tests. It ignored that NC BBCE removes the asset test, and that $1,425/month gross (64% FPL) and $931 net (42% FPL) both pass easily, giving $505/month."
-us,scenario_054,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model subtracted half of SE tax from gross income ($1,324/month instead of the $1,425 SNAP counts) and used a $224 standard deduction and $789 maximum instead of $209 and $785. It then raised its own $6,456 figure to $7,152 with an unexplained COLA adjustment."
-us,scenario_054,snap,claude-opus-4.8,llm_error,other,False,"The model's own calculation reached about $508/month ($6,096/year), close to the correct $505. It then replaced that with an unsupported 'refined' $707/month, which inflated the annual figure to $8,484."
-us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model assumed an $800 maximum allotment and asserted $583/month without computing the contribution. The correct steps give $931 net income, a $280 contribution, and $785 − $280 = $505/month."
-us,scenario_054,snap,claude-opus-5.5,llm_error,other,False,"All inputs were correct ($785 max, $931 net). The model did not round the 30% contribution of $279.30 up to $280 as SNAP requires, so it reported $505.70 instead of $505/month."
-us,scenario_054,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied a 40% self-employment cost-of-doing-business deduction that PolicyEngine does not apply, which cut countable income to $855/month instead of $1,425. It also used a $228 standard deduction and a $766 maximum. Its reasoning concluded $7,548, yet it submitted $5,532, which none of its work supports."
-us,scenario_054,snap,claude-sonnet-5,llm_error,other,False,"The model estimated a contribution of about $280 against a $768 maximum, which gives about $488/month. It then wrote $350/month, an arithmetic error, on top of using a stale $768 maximum instead of $785."
-us,scenario_054,snap,claude-sonnet-5.5,llm_error,other,False,"The model correctly reached $931 net income and a $785 maximum. It left the 30% contribution unrounded ($279.30) instead of rounding it up to $280, which gave about $505.70/month rather than $505."
-us,scenario_054,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model subtracted $2,416 of SE tax from gross income, but SNAP counts the full $1,425/month. It also used a $198 standard deduction and a $775 maximum instead of $209 and $785, which inflated the benefit to $540.72/month."
-us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,The model used a $202 standard deduction and a $781 maximum allotment instead of $209 and $785. That produced $938 net income and $499/month instead of $931 net and $505/month.
-us,scenario_054,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model used a $208 standard deduction and a $784 maximum instead of $209 and $785. It also left the 30% contribution unrounded ($279.60 rather than rounding up), giving $504.40 instead of $505/month."
-us,scenario_054,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Net income of $931 was correct, but the model used a $787 maximum allotment instead of $785. It also did not round the $279.30 contribution up to $280, which gave $507.70/month instead of $505."
-us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model used an outdated $766 maximum allotment instead of $785 and a lower standard deduction that produced $942 net instead of $931. Its result was $483.40/month instead of $505.
-us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's $3,720 equals $310/month, about $195 below the correct $785 − $280 = $505. That figure is consistent with a badly understated maximum allotment or an overstated net income, not the $931 net that the $209 standard deduction and 20% earned income deduction produce."
-us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model's $5,800.80 equals $766 − 0.3 × $942 = $483.40/month. That reflects an outdated $766 maximum and a $198 standard deduction instead of the $785 maximum and $209 deduction that give $505/month."
-us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used a $747 maximum allotment and a $198 standard deduction instead of $785 and $209. That gave $464/month instead of $505.
-us,scenario_054,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model claimed income exceeds SNAP thresholds, but $1,425/month gross is 64% of the $2,220.83 monthly poverty guideline and $931 net is 42%. Both tests pass, so the household receives $505/month."
-us,scenario_054,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's $7,548 equals $629/month, the result of applying a 40% self-employment expense deduction (countable income $855) with a $228 standard deduction and a $766 maximum. SNAP here counts the full $1,425/month, giving $505/month."
-us,scenario_054,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $5,820 equals $485/month, consistent with the FY2025 $768 maximum and a lower standard deduction. The FY2026 $785 maximum and $209 deduction yield $505/month."
-us,scenario_054,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model's $5,844 equals $487/month = $768 − $281, which uses the FY2025 $768 maximum and $204 standard deduction instead of the FY2026 $785 maximum and $209 deduction."
-us,scenario_054,snap,gemini-4-argon,llm_error,thresholds_rates,False,"The model's $5,856 equals $488/month, consistent with subtracting the correct $280 contribution from the outdated $768 maximum allotment instead of the $785 maximum."
-us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"The model assumed a $9,600/year ($800/month) maximum allotment and a $228/month standard deduction instead of $785 and $209. It also computed the contribution on annual totals without the monthly round-up, giving $6,317."
-us,scenario_054,snap,glm-5.3,llm_error,thresholds_rates,False,"Net income of $931 was correct, but the model used an $816 maximum allotment instead of $785. That inflated the benefit to $537/month instead of $505."
-us,scenario_054,snap,gpt-5.4-mini,llm_error,other,False,"The model's $8,868 equals $739/month, nearly the full maximum allotment, which means it effectively skipped the 30% contribution on $931 of net income. The correct calculation is $785 − $280 = $505/month."
-us,scenario_054,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model declined to compute SNAP even though household size (3), $17,100 SE income and all other inputs were given. Those facts yield $931 net income and a $505 monthly allotment under NC BBCE."
-us,scenario_054,snap,gpt-5.5,llm_error,other,False,"The model correctly identified $785 maximum and $931 net, but stated $785 − 30% × $931 = $575, an arithmetic error. The correct figure is $785 − $280 (rounded-up contribution) = $505/month."
-us,scenario_054,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model applied a resource limit to the $14,000 bank balance. It missed that North Carolina's TANF non-cash broad-based categorical eligibility waives the SNAP asset test for this household."
-us,scenario_054,snap,gpt-5.6-sol,llm_error,other,False,"The model correctly reached $931 net income, then rounded the $505.70 result up to $506. SNAP rounds the 30% contribution up ($279.30 → $280), which makes the allotment $505."
-us,scenario_054,snap,gpt-5.6-terra,llm_error,other,False,"Using $785 and $931 correctly, the model rounded the benefit up to $506. SNAP rounds the expected contribution up to $280, so the allotment is $505/month."
-us,scenario_054,snap,gpt-6-astra,llm_error,other,False,The formula 785 − 0.30 × 931 was evaluated without rounding the $279.30 contribution up to $280. That produced $505.70/month instead of $505.
-us,scenario_054,snap,gpt-6-luna,llm_error,other,False,"The model kept the unrounded $505.70/month instead of rounding the 30% contribution up to $280. The correct allotment is $505/month, or $6,060/year."
-us,scenario_054,snap,gpt-6.1-terra,llm_error,other,False,"The model's $6,068.40 is 12 × $505.70, which shows it never applied the SNAP rule that rounds the 30% net-income contribution up to the next whole dollar ($280). That rule gives $505/month."
-us,scenario_054,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model claimed income and assets exceed the limits. NC BBCE waives the asset test, and $1,425/month gross (64% FPL) passes the income tests, giving $505/month."
-us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used a $766/month maximum ($9,192/year) and a $198 standard deduction instead of $785 and $209. That understated the benefit at $5,801."
-us,scenario_054,snap,grok-4.6,llm_error,thresholds_rates,False,"The model used FY2025 parameters, a $768 maximum allotment and a $204 standard deduction, instead of the $785 maximum and $209 deduction that give $505/month."
-us,scenario_054,snap,grok-4.7,llm_error,thresholds_rates,False,"The model applied the FY2025 $768 maximum and $204 standard deduction instead of $785 and $209. It also computed the contribution annually without the monthly round-up, giving $5,846.40."
-us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,The model assumed an $811 maximum allotment instead of $785 and a standard deduction of about $205 instead of $209. That inflated the benefit to $531/month.
-us,scenario_054,snap,inkling,llm_error,categorical_eligibility,False,"The model zeroed SNAP because of the $14,000 bank balance, missing that NC BBCE through TANF non-cash eligibility removes the asset test."
-us,scenario_054,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model applied a countable resource limit for non-elderly/disabled households. North Carolina's broad-based categorical eligibility waives that limit, so the $14,000 bank balance does not disqualify the household."
-us,scenario_054,snap,kimi-k3,llm_error,thresholds_rates,False,"Net income ($931) and the rounded-up $280 contribution were both correct, but the model used a $783 maximum allotment instead of $785. That gave $503/month instead of $505."
-us,scenario_054,snap,minimax-m3,llm_error,thresholds_rates,False,"The model claimed $17,100 exceeds 130% FPL (about $26,000), which contradicts its own numbers since $17,100 is below $26,000. Gross income is 64% FPL and the household receives $505/month."
-us,scenario_054,snap,muse-spark-1.3,llm_error,thresholds_rates,False,The model used about $940 net (standard deduction too low) and an approximate $788 maximum instead of $931 and $785. Those approximations gave $506 instead of $785 − $280 = $505/month.
-us,scenario_054,snap,muse-wonder,llm_error,thresholds_rates,False,The model used the outdated $768 maximum allotment and a lower standard deduction (net $936) instead of $785 and $209 (net $931). That gave $487/month instead of $505.
-us,scenario_054,snap,ox-alpha,llm_error,other,False,"All inputs were correct ($785 max, $209 standard deduction, $931 net). The model left the 30% contribution unrounded at $279.30 instead of rounding up to $280, which yielded $505.70 rather than $505/month."
-us,scenario_054,snap,qwen-3.7-max,llm_error,other,False,"The model treated the household as receiving a maximum benefit of $934/month and never subtracted 30% of its $931 net income. The maximum for 3 is $785, and the $280 contribution reduces the benefit to $505."
-us,scenario_054,snap,qwen3.8-max,llm_error,thresholds_rates,False,"The model misstated monthly earnings as $1,579 instead of $1,425 and claimed net income exceeds the limit. Net income of $931 is only 42% of the poverty guideline and yields $505/month."
+us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,It projected a 3-person maximum allotment of about $820 instead of $785 and used a $211 standard deduction instead of $209. That inflated the benefit to $540 a month; the correct figure is $785 − $280 = $505.
+us,scenario_054,snap,claude-fable-5.1,llm_error,other,False,"Its inputs were correct ($785 maximum, $931 net income), but it left the expected contribution unrounded at $279.30. SNAP rounds 30% of net income up to the next whole dollar ($280), so the monthly benefit is $505, not $505.70."
+us,scenario_054,snap,claude-fable-5.2,llm_error,other,False,"It computed net income of $931 and used the $785 maximum, but kept 30% × 931 at $279.30. Rounding the contribution up to $280 gives a whole-dollar benefit of $505 a month, or $6,060 a year, rather than $6,068.40."
+us,scenario_054,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It applied a SNAP asset limit to the $14,000 bank balance. North Carolina's broad-based categorical eligibility (TANF non-cash) waives the asset test, so the household gets $505 a month."
+us,scenario_054,snap,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It treated the $14,000 in assets and the income as disqualifying. BBCE waives the asset test, gross income is only 64% of the poverty line, and net income of $931 leaves a $505 monthly benefit."
+us,scenario_054,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It cut SNAP income by half of SE tax, though SNAP counts the full $1,425 a month, and used a $224 standard deduction and a $789 maximum. It then raised its own $6,456 result to $7,152 with an unsupported 'COLA adjustment'."
+us,scenario_054,snap,claude-opus-4.8,llm_error,other,False,"It reached roughly $508 a month ($6,096 a year), then replaced that with an unexplained $707 a month. The correct calculation is $785 − $280 = $505 a month."
+us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,"It assumed an $800 maximum allotment and an unsupported $583 monthly benefit. With the $785 maximum and a $280 contribution on $931 of net income, the benefit is $505 a month."
+us,scenario_054,snap,claude-opus-5.5,llm_error,other,False,"Its deductions and maximum allotment were correct, but it did not round 30% of $931 ($279.30) up to $280. That left it at $505.70 a month instead of the whole-dollar $505."
+us,scenario_054,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It invented a 40% self-employment cost deduction (countable income $855 a month instead of $1,425) and used outdated $228 standard deduction and $766 maximum figures. It also submitted $5,532, which does not match its own $7,548 calculation."
+us,scenario_054,snap,claude-sonnet-5,llm_error,other,False,"It used a stale $768 maximum, and $768 minus its own $280 contribution is $488, not the $350 it reported. The correct result is $785 − $280 = $505 a month."
+us,scenario_054,snap,claude-sonnet-5.5,llm_error,other,False,"It used the right $785 maximum and $931 net income, but annualized the unrounded $505.70. SNAP rounds the contribution up to $280, giving $505 a month and $6,060 a year."
+us,scenario_054,snap,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted $2,416 of SE tax from income, but SNAP counts the full $1,425 a month of self-employment income. It also used stale figures: a $198 standard deduction and a $775 maximum allotment."
+us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,It used a $781 maximum allotment and a $202 standard deduction instead of the FY2026 values of $785 and $209. That gave $499 a month instead of $505.
+us,scenario_054,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $784 maximum and a $208 standard deduction instead of $785 and $209, and left the contribution unrounded. The correct figures are a $280 contribution and a $505 monthly benefit."
+us,scenario_054,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Its deductions and $931 net income were correct, but it used a $787 maximum allotment instead of $785 and did not round the $279.30 contribution up to $280. The correct monthly benefit is $505, not $507.70."
+us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used an outdated $766 maximum allotment and $942 of net income, which reflects an outdated $198 standard deduction. The FY2026 values are $785 and $209, which give $931 of net income and a $505 benefit."
+us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no derivation, and its $310 a month is far below the correct $505 ($785 maximum minus a $280 contribution on $931 of net income). That gap is consistent with a badly understated maximum allotment or an overstated contribution."
+us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $483.40 a month equals $766 − 30% × $942, so it used an outdated $766 maximum and $198 standard deduction. The FY2026 values of $785 and $209 give $505 a month."
+us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,It used a $747 maximum allotment and a $198 standard deduction instead of the FY2026 values of $785 and $209. That gave $464 a month instead of $505.
+us,scenario_054,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It claimed income exceeds SNAP limits, but gross income of $1,425 a month is 64% of the poverty line and net income is 42%. Both tests pass, and the household gets $505 a month."
+us,scenario_054,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $629 a month matches $766 − 30% × $456, which implies a 40% self-employment expense deduction that understates countable income. SNAP counts the full $1,425 a month, giving $931 of net income and a $505 benefit."
+us,scenario_054,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $485 a month reflects an outdated maximum allotment of about $766 rather than the FY2026 value of $785. With $785 minus a $280 contribution, the benefit is $505 a month."
+us,scenario_054,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $487 a month equals the FY2025 $768 maximum minus a $281 contribution based on the FY2025 $204 standard deduction. The FY2026 values ($785 maximum, $209 deduction) give $505 a month."
+us,scenario_054,snap,gemini-4-argon,llm_error,thresholds_rates,False,"Its $488 a month equals the FY2025 $768 maximum minus a $280 contribution. The applicable FY2026 maximum is $785, giving $505 a month."
+us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"It assumed an $800 monthly maximum ($9,600 a year) and a $228 monthly standard deduction instead of $785 and $209. That overstated the benefit; the correct annual figure is $6,060."
+us,scenario_054,snap,glm-5.3,llm_error,thresholds_rates,False,"Its net income of $931 was correct, but it assumed an $816 maximum allotment instead of $785. That inflated the benefit to $537 a month instead of $505."
+us,scenario_054,snap,gpt-5.4-mini,llm_error,other,False,"It called countable income 'very low' and gave $739 a month, almost ignoring the 30% contribution. Net income is $931, so the contribution is $280 and the benefit is $785 − $280 = $505."
+us,scenario_054,snap,gpt-5.4-nano,llm_error,other,False,"It wrongly claimed income and household size were not provided and defaulted to $0. A 3-person household with $1,425 of monthly self-employment income gets $505 a month after the standard deduction, the earned income deduction and the 30% contribution."
+us,scenario_054,snap,gpt-5.5,llm_error,other,False,"Its inputs were correct ($785 maximum, $931 net income), but it miscalculated $785 − 30% × $931 as $575. The correct result is $785 − $280 = $505 a month."
+us,scenario_054,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It applied a resource limit to the $14,000 bank balance. North Carolina's TANF non-cash categorical eligibility waives the SNAP asset test, so the benefit is $505 a month."
+us,scenario_054,snap,gpt-5.6-sol,llm_error,other,False,"It rounded the $505.70 benefit up to $506 a month. SNAP rounds the 30% contribution up to $280, which gives exactly $505 a month and $6,060 a year."
+us,scenario_054,snap,gpt-5.6-terra,llm_error,other,False,"It rounded $785 − $279.30 up to $506. The contribution rounds up to $280, so the benefit is $505 a month."
+us,scenario_054,snap,gpt-6-astra,llm_error,other,False,"Its formula used the right $785, $209 and 20% inputs but kept the contribution at $279.30. Rounding it up to $280 gives a whole-dollar benefit of $505 a month."
+us,scenario_054,snap,gpt-6-luna,llm_error,other,False,"It annualized the unrounded $505.70. The 30% contribution rounds up to $280, so the benefit is $505 a month and $6,060 a year."
+us,scenario_054,snap,gpt-6.1-terra,llm_error,other,False,"Its $6,068.40 annualizes $505.70 because it did not round the $279.30 contribution up to $280. The correct benefit is $505 a month."
+us,scenario_054,snap,grok-4.3,llm_error,categorical_eligibility,False,"It claimed income and assets exceed SNAP limits. BBCE waives the asset test and gross income is only 64% of the poverty line, so the household gets $505 a month."
+us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"It used an outdated $766 monthly maximum ($9,192 a year) and a $198 monthly standard deduction ($2,376 a year) instead of the FY2026 values of $785 and $209. That understated the benefit."
+us,scenario_054,snap,grok-4.6,llm_error,thresholds_rates,False,"It used the FY2025 values: a $768 monthly maximum ($9,216 a year) and a $204 standard deduction. The FY2026 values of $785 and $209 give $505 a month."
+us,scenario_054,snap,grok-4.7,llm_error,thresholds_rates,False,"It used the FY2025 $768 maximum and $204 standard deduction instead of the FY2026 values of $785 and $209. That understated the benefit, which is correctly $505 a month."
+us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,It assumed an $811 maximum allotment and a standard deduction of about $205 instead of $785 and $209. That inflated the benefit to $531 a month instead of $505.
+us,scenario_054,snap,inkling,llm_error,categorical_eligibility,False,"It treated the $14,000 bank balance as exceeding a resource limit. North Carolina's broad-based categorical eligibility waives the SNAP asset test, so the household gets $505 a month."
+us,scenario_054,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"It applied the standard countable-resource limit for households without elderly or disabled members. TANF non-cash categorical eligibility in North Carolina removes that asset test, so the household gets $505 a month."
+us,scenario_054,snap,kimi-k3,llm_error,thresholds_rates,False,"Its deductions, $931 net income and $280 rounded contribution were correct, but it used a $783 maximum allotment instead of $785. That gave $503 a month instead of $505."
+us,scenario_054,snap,minimax-m3,llm_error,thresholds_rates,False,"It said $17,100 of income exceeds a 130% poverty limit of about $26,000, but $17,100 is below that limit. The household passes the gross and net income tests and gets $505 a month."
+us,scenario_054,snap,muse-spark-1.3,llm_error,thresholds_rates,False,It used a maximum of about $788 instead of $785 and net income of about $940 instead of $931 (with a $209 standard deduction). It also rounded to $506 a month instead of $505.
+us,scenario_054,snap,muse-wonder,llm_error,thresholds_rates,False,"It used the FY2025 $768 maximum and a stale standard deduction, which put net income at $936. The FY2026 values ($785 maximum, $209 deduction) give $931 of net income and a $505 benefit."
+us,scenario_054,snap,ox-alpha,llm_error,other,False,"Its inputs ($785 maximum, $931 net income) were correct, but it left the contribution at $279.30. Rounding it up to $280 gives $505 a month and $6,060 a year."
+us,scenario_054,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It gave the household a $934 maximum benefit and ignored the 30% net-income contribution. The FY2026 3-person maximum is $785, and $931 of net income reduces it by $280, to $505 a month."
+us,scenario_054,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It misstated monthly earnings as $1,579 instead of $1,425 and claimed net income exceeds the limit. Net income of $931 is 42% of the poverty line, well under 100%, so the household gets $505 a month."
us,scenario_054,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model correctly applied North Carolina's $19,125 head-of-household standard deduction against federal AGI of $15,892 and stated twice in its own reasoning that NC taxable income floors at $0 and ""NC tax = 0,"" then discarded that conclusion and submitted $245. No NC computation produces $245 from these facts: at the 2026 flat rate of 3.99% it implies $6,140 of taxable income, which the household does not have once the standard deduction is applied."
us,scenario_054,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"After correctly computing $0 NC taxable income using the head-of-household standard deduction of $19,125, the model reassigned the filer to single status and its $12,750 standard deduction despite two qualifying children in the household, yielding $3,142 of taxable income and $133.50 of tax. It then submitted $596, a figure tied to neither derivation and roughly 3.75% of gross self-employment income with no standard deduction at all."
us,scenario_054,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model applied the $12,750 single-filer standard deduction rather than the $19,125 head-of-household deduction the head qualifies for with two dependent children, and used the 2025 rate rather than the 2026 flat rate of 3.99%: $15,892 − $12,750 = $3,142 taxed at 4.5% is $141.39, matching its $142. Under the correct head-of-household deduction the base is zero and the tax is $0."
@@ -5208,113 +5268,115 @@ us,scenario_056,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"T
us,scenario_056,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for self_employment_tax, so its answer for this output was missing. The correct value is $4,340 × 0.9235 × 0.153 = $613.22."
us,scenario_056,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model claimed $4,340 of SE income yields net SE earnings below the $400 threshold. That comparison is wrong: net SE earnings are $4,007.99, about ten times the $400 floor. The full 15.3% SE tax therefore applies, giving $613.22."
us,scenario_056,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model stated that $4,340 of net self-employment income is below the $400 threshold, which is a basic numeric comparison error. Net SE earnings of $4,007.99 ($4,340 × 0.9235) exceed $400, so SE tax at 15.3% is $613.22."
-us,scenario_056,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"Added $564/month of mortgage interest to the $996/month in property taxes, which pushed excess shelter to the roughly $744 cap and gave $159/month. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Counted the $18,722 of mortgage interest plus property tax as shelter cost and hit the $8,928 annual shelter cap, giving net income of $5,535. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"Treated $17,839 as over the income limit and the mortgage balance as a disqualifying asset. Both income tests and the asset test pass in the frozen derivation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"Never ran the benefit formula and returned $0 for a household that passes the gross and net income tests. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"Used a net income limit of about $1,255 and omitted the shelter deduction used in the frozen calculation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"Acknowledged that shelter deductions could permit eligibility, then returned $0 based on assets 'implied' by home ownership and financial assistance. The home is not a countable resource, and the household passes the asset test. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Said $17,839 of income exceeds the gross and net limits. Both income tests pass in the frozen derivation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Left out the $12,000 of financial assistance and used only $487/month of income, which drove net income to $0 and gave the full $298 maximum. The frozen derivation counts the financial assistance as unearned income. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"Found that both income tests pass, then returned $0 after omitting the shelter deduction and state minimum used in the frozen calculation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"Said countable income exceeds 130% of the poverty level and treated financial assistance as an asset problem. Both income tests pass in the frozen derivation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Explicitly left the $12,000 of financial assistance out of income, which brought countable income down to $487/month and gave the $298 maximum. The frozen derivation counts the financial assistance as unearned income. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"Applied only the earned-income and standard deductions and skipped the shelter deduction used in the frozen calculation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Applied the able-bodied adult (ABAWD) time limit to reduce eligibility to zero months. The frozen $1,140 reference assumes 40 weekly work hours. The developer decision recognizes the zero-hours reading and excludes this output because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Applied a capped $712 excess shelter deduction built from mortgage interest plus property tax, giving $142/month. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"Attributed $0 to 30% of net income exceeding the maximum allotment without showing deductions. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Built a capped $672 shelter deduction from mortgage interest, property tax and utility expenses that were never listed, giving $127.72/month. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Said gross income exceeds the SNAP threshold. The gross income test passes in the frozen derivation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"Applied the maximum capped excess shelter deduction using mortgage interest plus property taxes, which left net income at about $489 and gave $158/month. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"Listed self-employment and alimony as countable income and estimated a $292 monthly maximum without explaining its treatment of financial assistance. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Returned $0 without doing any calculation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Its $140.72/month result implies a capped shelter deduction of about $700, which is only reachable by adding mortgage interest to property taxes. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"Returned $3,504 annually, equal to $292 per month, without a detailed income or shelter calculation. The held FY2026 one-person maximum is $298 per month. The frozen $1,140 reference remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Ruled the household ineligible under the able-bodied adult (ABAWD) work rule. The frozen $1,140 reference assumes 40 weekly work hours. The developer decision recognizes the zero-hours reading and excludes this output because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,gemini-4-argon,prompt_ambiguity,categorical_eligibility,False,"Counted only $13,499 of unearned income and dropped the $4,340 of self-employment income ($361.67/month of earned income). It also capped shelter at $744 using mortgage interest, reaching about $172 of net income. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Added $564.08/month of mortgage interest to $995.75 in property taxes and capped excess shelter at $733, giving $159.63/month. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"Excluded the $12,000 of financial assistance as welfare income, skipped the 20% earned-income deduction and ignored the property-tax shelter deduction. The frozen derivation counts the financial assistance as income. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Assumed $0 without running the SNAP formula. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Assumed $0 without computing eligibility or the allotment. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Applied $23/month as the federal minimum; the held FY2026 federal minimum is $24/month. The frozen derivation's $63.18 formula benefit exceeds that federal minimum. The frozen $1,140 reference remains excluded because work hours and mortgage residence are unlisted."
-us,scenario_056,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"Attributed $0 to high net income without showing deductions. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Used a capped excess shelter deduction that includes mortgage interest, giving $159/month. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Built a capped excess shelter deduction from mortgage interest plus property taxes, giving $158/month. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"Applied the $8,928 annual shelter cap, which is reachable only by adding mortgage interest to property taxes, and got net income of $5,535. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"Returned $0 because 30% of net income exceeded the one-person maximum, without showing deductions. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Included mortgage interest in shelter costs and hit the $8,928 annual cap, giving a benefit of $1,915.50. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,gpt-6.1-terra,prompt_ambiguity,categorical_eligibility,False,"Its $1,915.50 matches the capped excess shelter deduction built from mortgage interest plus property taxes. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"Said income and assets make the household ineligible. The income and asset tests pass in the frozen derivation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Acknowledged that income is within limits and returned $0, citing incomplete shelter and utility details. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"Returned $0 on the ground that net income was too high with no listed rent or utilities. The frozen calculation uses the listed real estate taxes as shelter costs. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Counted only alimony and self-employment income, dropping the $12,000 of financial assistance, and set shelter deductions to $0 despite the listed real estate taxes. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Used an annual standard deduction of about $3,492 and omitted the shelter deduction used in the frozen calculation. The held FY2026 standard deduction is $209/month. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"Applied a capped $712 shelter deduction built from mortgage interest plus property taxes, giving $144/month. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no SNAP value or explanation, so there was no answer to score against the $1,140 annual benefit (New Jersey's $95/month minimum)."
-us,scenario_056,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"Combined $1,560.17 of mortgage interest and real estate taxes as shelter cost and capped excess shelter at $744, giving $158.13/month. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Said income exceeds the one-person limits. Both income tests pass in the frozen derivation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,muse-spark-1.3,prompt_ambiguity,categorical_eligibility,False,"Applied $23/month after computing a negative formula result; the held FY2026 federal minimum is $24/month. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,muse-wonder,prompt_ambiguity,categorical_eligibility,False,"Applied only the earned-income and standard deductions, leaving out the shelter deduction used in the frozen calculation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Capped excess shelter using $564/month of mortgage interest plus $996 of property taxes, giving about $150/month. The frozen reference uses property taxes alone as shelter costs. The occupied-home reading counts mortgage interest as a shelter cost. The $1,140 annual reference assumes 40 weekly work hours and remains excluded because hours and mortgage residence are unlisted."
-us,scenario_056,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"Contradicted itself: it said $17,839 exceeds a gross limit of about $19,000, then returned $0 with no shelter deduction. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
-us,scenario_056,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Said gross income exceeds 130% of the poverty level and net income exceeds 100%. Both income tests pass in the frozen derivation. The frozen $1,140 annual reference assumes 40 weekly work hours and property taxes alone as shelter costs. The output remains excluded because work hours and occupied-home mortgage status are unlisted."
+us,scenario_056,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"Added the $564/month home mortgage interest to the $996/month real estate taxes as shelter costs, so the excess shelter deduction hit the ~$744 cap, net income fell to $462, and the benefit came out to $159/month. PolicyEngine's shelter cost counts only the $995.75 in real estate taxes. That gives an uncapped $395.49 excess shelter deduction, $805.03 net income, and a $63.18 formula benefit, which New Jersey's $95 minimum raises to $1,140/yr."
+us,scenario_056,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Treated mortgage interest plus property tax ($18,722/yr) as shelter cost and applied the $8,928 annual shelter cap, giving $5,535 net income. Only the $11,949 in real estate taxes counts, so the excess shelter deduction is an uncapped $395.49/month and net income is $805.03/month. The $63.18 formula benefit is then raised to New Jersey's $95 monthly minimum."
+us,scenario_056,snap,claude-fable-5.2,prompt_ambiguity,categorical_eligibility,False,"Set SNAP to zero by assuming the 20-year-old fails work requirements. PolicyEngine finds that the household passes the work requirement test because the head has self-employment earnings. The household is eligible, and its $63.18 formula benefit is raised to New Jersey's $95 monthly minimum."
+us,scenario_056,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"Said $17,839 of income exceeds the SNAP limit and treated the $100,341 mortgage balance as an asset. Gross income of $1,486.54/month is only 1.12× the poverty guideline, which passes, and a mortgage balance is debt, not a countable resource. With deductions, net income is $805.03, and New Jersey's $95 minimum gives $1,140/yr."
+us,scenario_056,snap,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"Returned $0 without computing a benefit, even though the household passes the gross test (1.12× poverty) and the net test (0.61×). Subtracting the $72.33 earned-income, $213.68 standard, and $395.49 excess shelter deductions from $1,486.54 gives $805.03 net income. The resulting $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"Skipped the excess shelter deduction for the $995.75/month in real estate taxes. That left net income at about $1,400, above the net limit, so it zeroed the benefit. With the $395.49 shelter deduction, net income is $805.03 (0.61× poverty), and the $63.18 formula benefit is raised to New Jersey's $95 monthly minimum."
+us,scenario_056,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"Agreed that shelter deductions qualify the household, then zeroed the benefit based on assets implied by home ownership. The home is an excluded resource, and PolicyEngine finds that the household passes the asset test. The eligible one-person unit's $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Asserted that $17,839 of countable income exceeds the gross and net limits. Gross income of $1,486.54/month is 1.12× the $1,330 guideline, and net income after the standard, earned-income, and $395.49 excess shelter deductions is $805.03 (0.61×), so both tests pass. The benefit is New Jersey's $95 monthly minimum."
+us,scenario_056,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Left the $12,000 of financial assistance out of countable income ($487/month instead of $1,486.54) and counted mortgage interest as shelter cost, which pushed net income to $0 and gave the $298 maximum. PolicyEngine counts financial assistance as unearned income and uses only real estate taxes as shelter cost. That gives $805.03 net income, a $63.18 formula benefit, and New Jersey's $95 minimum."
+us,scenario_056,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"Computed $1,216 net income with no excess shelter deduction for the $995.75/month in real estate taxes. It then set the negative formula result to $0, even though it found the household passes both income tests. An eligible one-person household gets the minimum allotment, which New Jersey sets at $95/month; with the $395.49 shelter deduction the formula gives $63.18, still below that minimum."
+us,scenario_056,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"Claimed countable income exceeds 130% of poverty, but gross income of $1,486.54/month is only 1.12× the $1,330 guideline. After deductions, net income is $805.03 (0.61×), and the $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Explicitly excluded the $12,000 of financial assistance from income and added mortgage interest to shelter costs, which drove net income to near zero and gave the $298 maximum. PolicyEngine counts the financial assistance (gross $1,486.54/month) and only the $995.75 in real estate taxes as shelter cost. That yields $805.03 net income, a $63.18 formula benefit, and New Jersey's $95 minimum."
+us,scenario_056,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"Concluded that 30% of net income after only the earned-income and standard deductions wipes out the maximum allotment. It skipped the $395.49 excess shelter deduction from real estate taxes and the minimum benefit owed to an eligible one-person household. Net income is $805.03, the formula gives $63.18, and New Jersey's $95 minimum yields $1,140/yr."
+us,scenario_056,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Applied the ABAWD time limit to cut benefits to zero months. PolicyEngine finds that the household meets SNAP work requirements, so no time-limit disqualification applies. The eligible household's $63.18 formula benefit is raised to New Jersey's $95 monthly minimum."
+us,scenario_056,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Counted mortgage interest as a shelter cost, so the shelter deduction hit the $712 cap and net income fell to $498.25, giving $142/month. Only the $995.75/month in real estate taxes counts, so the excess shelter deduction is $395.49 and net income is $805.03. The $63.18 formula benefit is then raised to New Jersey's $95 minimum."
+us,scenario_056,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"Claimed 30% of net income exceeds the one-person maximum. That holds only if the $395.49 excess shelter deduction from real estate taxes is ignored, and even then an eligible one-person household gets the minimum allotment, not $0. With $805.03 net income the formula gives $63.18, which New Jersey's $95 minimum raises."
+us,scenario_056,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Built shelter costs from mortgage interest, property taxes, and unlisted utilities, hitting a $672 cap with $544.25 net income and a $127.72/month benefit. PolicyEngine counts only the $995.75 in real estate taxes, which gives a $395.49 excess shelter deduction and $805.03 net income. The resulting $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Asserted that gross income exceeds SNAP limits, but $1,486.54/month is only 1.12× the poverty guideline and passes the gross test. The household is eligible, and its $63.18 formula benefit is raised to New Jersey's $95 monthly minimum."
+us,scenario_056,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"Applied the maximum excess shelter deduction by counting mortgage interest alongside property taxes, which gave $489 net income and $158/month. Only the $995.75/month in real estate taxes is a shelter cost, so the deduction is an uncapped $395.49 and net income is $805.03. The $63.18 formula benefit becomes New Jersey's $95 minimum."
+us,scenario_056,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"Counted only self-employment and alimony income, leaving out the $1,000/month of financial assistance, and used capped shelter costs that included mortgage interest. That produced the full $292 maximum. PolicyEngine counts the financial assistance as unearned income and only real estate taxes as shelter cost, giving $805.03 net income, a $63.18 formula benefit, and New Jersey's $95 minimum."
+us,scenario_056,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Returned $0 with no computation. The household passes the gross (1.12× poverty) and net (0.61×) tests, and its $63.18 formula benefit is raised to New Jersey's $95 minimum, which gives $1,140/yr."
+us,scenario_056,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"The $140.72/month answer matches counting mortgage interest plus property taxes so that the excess shelter deduction hits the cap. PolicyEngine counts only the $995.75 in real estate taxes, which gives a $395.49 deduction and $805.03 net income. The $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"$3,504 is the full $292 maximum allotment, which implies zero net income. That only happens if the $1,000/month of financial assistance is dropped and shelter deductions are overstated. Correctly, net income is $805.03, the formula benefit is $63.18, and New Jersey's $95 minimum applies."
+us,scenario_056,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Treated the 20-year-old as an ABAWD who fails work rules. PolicyEngine finds that the household passes the work requirement test, given the head's self-employment earnings. The eligible household's $63.18 formula benefit is raised to New Jersey's $95 monthly minimum."
+us,scenario_056,snap,gemini-4-argon,prompt_ambiguity,categorical_eligibility,False,"Counted only the $13,499 of unearned income, leaving out the $361.67/month of self-employment earnings, and added mortgage interest to shelter costs so the deduction hit the cap. That gave about $172 net income and $246/month. Correctly, gross income is $1,486.54, the shelter deduction from real estate taxes alone is $395.49, net income is $805.03, and New Jersey's $95 minimum applies."
+us,scenario_056,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Added $564.08 of monthly mortgage interest to the $995.75 in property taxes, so the excess shelter deduction hit the $733 cap and net income fell to $471.25. Only property taxes count, so the excess shelter deduction is $395.49 and net income is $805.03. The $63.18 formula benefit is then raised to New Jersey's $95 minimum."
+us,scenario_056,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"Excluded the $12,000 of financial assistance as TANF-type welfare and took no shelter deduction, giving $283 net income and $207/month. PolicyEngine counts financial assistance as unearned income (gross $1,486.54) and deducts $395.49 of excess shelter cost from the real estate taxes. That yields $805.03 net income, a $63.18 formula benefit, and New Jersey's $95 minimum."
+us,scenario_056,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Defaulted to $0 on the claim that the facts do not support a benefit. The listed income and real estate taxes fully determine one: net income is $805.03 and the formula benefit is $63.18, which New Jersey's $95 minimum raises to $1,140/yr."
+us,scenario_056,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Defaulted to $0, claiming there were no low-income indicators, even though gross income is only 1.12× the poverty guideline. After the earned-income, standard, and $395.49 excess shelter deductions, net income is $805.03, and the $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Correctly found the household eligible with a formula benefit below the minimum. It then applied the federal $23 one-person minimum instead of New Jersey's state-funded $95 monthly minimum, giving $276 instead of $1,140."
+us,scenario_056,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"Concluded the allotment is zero because net income is too high. It missed the $395.49 excess shelter deduction for the $995.75/month in real estate taxes and the minimum benefit for eligible one-person households. With $805.03 net income the formula gives $63.18, which New Jersey's $95 minimum raises."
+us,scenario_056,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Applied the capped excess shelter deduction by counting mortgage interest as a shelter cost, which gave $159/month. Only the $995.75/month in real estate taxes counts, so the excess shelter deduction is an uncapped $395.49 and net income is $805.03. The $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Counted listed mortgage interest plus property taxes as shelter cost and applied the shelter cap, which gave $158/month. PolicyEngine counts only real estate taxes, so the excess shelter deduction is $395.49, net income is $805.03, and the $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"Used the $8,928 annual shelter cap because mortgage interest was counted with property taxes, which gave $5,535 annual net income. Only the $11,949 in real estate taxes is shelter cost, so the excess shelter deduction is $395.49/month and net income is $805.03/month. The $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"Zeroed the benefit, claiming the 30% contribution exceeds the maximum. It ignored the $395.49 excess shelter deduction from real estate taxes and the minimum allotment owed to an eligible one-person household. Net income is $805.03 and the formula gives $63.18, which New Jersey's $95 minimum raises to $1,140/yr."
+us,scenario_056,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Counted mortgage interest as a shelter expense, so the excess shelter deduction hit the $8,928 annual cap and annual net income came out to $5,535. Only real estate taxes count, so the excess shelter deduction is $395.49/month and net income is $805.03/month. The $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,gpt-6.1-terra,prompt_ambiguity,categorical_eligibility,False,"Applied a capped shelter deduction that included mortgage interest, which gave $1,915.50/yr. PolicyEngine's shelter cost is only the $995.75/month in real estate taxes, so the deduction is $395.49 and net income is $805.03. The $63.18 formula benefit is raised to New Jersey's $95 monthly minimum."
+us,scenario_056,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"Asserted that income and assets make the household ineligible. Gross income is only 1.12× poverty and the household passes the asset test, since the home is an excluded resource. The eligible unit's $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Zeroed the benefit, citing incomplete shelter details, while also agreeing that income is within limits. The listed $11,949 in real estate taxes alone produce a $395.49 monthly excess shelter deduction. An eligible one-person household gets at least the minimum, which New Jersey sets at $95/month."
+us,scenario_056,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"Treated the lack of rent or utility inputs as zero shelter costs, ignoring the $11,949 in real estate taxes, and set the benefit to $0 instead of the minimum allotment. Property taxes give a $395.49 excess shelter deduction and $805.03 net income. The $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Left the $12,000 of financial assistance out of countable income and took no shelter deduction for the real estate taxes, giving $210.25 net income and $229/month. PolicyEngine counts gross income of $1,486.54/month and a $395.49 excess shelter deduction, giving $805.03 net income. The $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Took no excess shelter deduction (net income $13,479/yr) and zeroed the benefit when 30% of net income exceeded the maximum. The $995.75/month in real estate taxes give a $395.49 excess shelter deduction, bringing net income to $805.03/month. The eligible one-person household then receives New Jersey's $95 minimum."
+us,scenario_056,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"Applied a capped $712 shelter deduction by counting mortgage interest with property taxes, which gave $493 net income and $144/month. Only the $995.75 in real estate taxes counts, so the deduction is $395.49 and net income is $805.03. The $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no SNAP value or explanation, so there is no answer to score. The correct derivation gives $805.03 net income and a $63.18 formula benefit, which New Jersey's $95 minimum raises to $1,140/yr."
+us,scenario_056,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"Set shelter costs to $1,560.17 by adding mortgage interest to real estate taxes, which capped the excess shelter deduction at $744 and gave $466.25 net income. Shelter cost is only the $995.75 in real estate taxes, so the excess is an uncapped $395.49 and net income is $805.03. The $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Claimed income exceeds the one-person limits, but gross income of $1,486.54/month is 1.12× poverty and net income of $805.03 is 0.61× poverty, so both tests pass. The household receives New Jersey's $95 minimum benefit."
+us,scenario_056,snap,muse-spark-1.3,prompt_ambiguity,categorical_eligibility,False,"Correctly found the household eligible with a formula benefit below the minimum. It then applied the federal $23 minimum instead of New Jersey's $95 monthly state minimum, giving $276 instead of $1,140."
+us,scenario_056,snap,muse-wonder,prompt_ambiguity,categorical_eligibility,False,"Zeroed the benefit after only the earned-income and standard deductions, skipping the $395.49 excess shelter deduction from real estate taxes and the minimum allotment for an eligible one-person household. Net income is $805.03, the formula gives $63.18, and New Jersey's $95 minimum applies."
+us,scenario_056,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Applied the capped excess shelter deduction using $564/month of mortgage interest plus $996 of property taxes, which gave about $500 net income and $150/month. Only real estate taxes are shelter cost, so the deduction is $395.49 and net income is $805.03. The $63.18 formula benefit is raised to New Jersey's $95 minimum."
+us,scenario_056,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"Said income exceeds the gross limit while putting that limit at about $19,000, which is above its own $17,839 income total. The household passes at 1.12× poverty, net income after the shelter deduction is $805.03, and New Jersey's $95 minimum applies."
+us,scenario_056,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Claimed gross income exceeds 130% FPL, but $1,486.54/month is only 112% of the $1,330 guideline. With the $395.49 excess shelter deduction from real estate taxes, net income of $805.03 also passes. The resulting $63.18 formula benefit is raised to New Jersey's $95 minimum."
us,scenario_056,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_056,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"The model built the correct NJ gross income base of $5,839 but never applied New Jersey's $10,000 minimum filing threshold for single filers, which exempts this household's entire liability and drives the tax to $0.00. It compounded this by inventing a ""$2,000 standard deduction"" that New Jersey's gross income tax does not provide, then taxed the residual $2,839 at 1.4% to reach $39.75."
us,scenario_056,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model reproduced PolicyEngine's intermediate NJ main income tax exactly — $5,838.53 of gross income less the $1,000 regular exemption, taxed at the 1.4% first-bracket rate — and submitted that $67.75 as the final answer. It omitted the last step: New Jersey's minimum filing threshold exempts a single filer whose gross income is at or below $10,000, so the $67.74 preliminary liability is fully offset to $0.00."
us,scenario_056,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model swept the $12,000 of financial assistance into NJ gross income to reach $17,839, which New Jersey does not tax, and then claimed a $1,428 charitable deduction that the New Jersey gross income tax does not allow at all. Its residual $3,462 taxed at 1.4% yields $48.47, and it never reached the dispositive rule — the $10,000 single-filer minimum filing threshold that zeroes the liability on the true $5,838.53 gross income base."
us,scenario_056,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so the required key was absent from the submitted outputs. This is a contract failure rather than a substantive modeling error: the correct derivation is $5,838.53 of NJ gross income less the $1,000 regular exemption at 1.4% = $67.74, fully exempted by New Jersey's $10,000 minimum filing threshold to $0.00."
us,scenario_056,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the $12,000 of financial assistance as New Jersey gross income, inflating the base to $13,772.82 after the $1,000 exemption, when NJ gross income is only the $1,498.53 alimony plus $4,340.00 self-employment income. It then abandoned its own $192.82 arithmetic and submitted $299.43, a figure no step in its reasoning produces, and it never applied New Jersey's $10,000 single-filer minimum filing threshold, which exempts the entire $67.74 preliminary liability to $0.00."
-us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"Applied the federal age-25 minimum for childless EITC to New Jersey and concluded the NJ EITC is $0. NJ extends its EITC to childless filers aged 18 and older, so the 20-year-old qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"Recognized NJ's age-18 childless EITC rule and calculated a $308.55 federal-equivalent phase-in on $4,033.39 of net SE earnings, then took 40% to get $123.42. For this age-only eligibility pathway, the engine pays 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"Asserted that New Jersey has no state EITC or other refundable credits. NJ has a refundable EITC that covers childless filers from age 18, so this 20-year-old with $4,033.39 of net SE earnings qualifies."
-us,scenario_056,state_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"Treated NJ refundable credits as depending on NJ tax liability and qualifying children. The NJ EITC is refundable without any liability and covers childless filers aged 18+, so the head's $4,033.39 of net SE earnings produces a credit."
-us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Used a minimum age of 21 for the NJ childless EITC. NJ lowered that minimum to 18, so the 20-year-old head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Concluded that unmarried-partner status, age 20, and unearned income rule out any credit. Alimony is not investment income for the EITC disqualification test, and NJ extends its childless EITC to filers aged 18+, so the $4,033.39 of net SE earnings qualifies."
-us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"Required federal EITC eligibility and applied the under-25 exclusion. NJ separately extends its EITC to childless filers aged 18–24 who fail the federal age test, so the head qualifies."
-us,scenario_056,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"Recognized NJ's age-18 childless rule, computed net earnings of $4,033.39 and a 7.65% federal-equivalent phase-in of $308.55, and took 40% to get $123.42. For this age-only eligibility pathway, the engine pays 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"Required federal EITC eligibility and applied the federal 25–64 age range. NJ extends its EITC to childless filers aged 18 and older, so the 20-year-old head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Speculated that the head is a dependent and applied the federal age-25 childless rule. The head files as the tax unit head, and NJ extends its childless EITC to filers aged 18+, so the credit applies."
-us,scenario_056,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"Claimed the $50 NJ property-tax credit as a refundable credit. For filers below the NJ filing threshold, this refundable credit requires being 65+, blind, or disabled, and the model missed the NJ EITC available to childless filers aged 18+."
-us,scenario_056,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"Made NJ's credit depend on federal EITC eligibility and denied it because of age 20. NJ extends its EITC to childless filers aged 18–24 who fail the federal age test, so the head qualifies."
-us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"Set NJ EITC equal to a share of a $0 federal EITC. NJ computes a federal-equivalent credit for childless filers aged 18+ who fail the federal age test, so the head receives a credit based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"Applied the 7.65% phase-in to gross SE income of $4,340 rather than to earned income net of half the SE tax ($4,033.39). It then multiplied by 40%, giving $132.80. For this age-only eligibility pathway, the engine pays 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"Derived the NJ EITC solely as 40% of a $0 federal EITC. NJ extends its EITC to childless filers aged 18+ who fail the federal age-25 test, so the head qualifies."
-us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"Required federal EITC eligibility for the NJ EITC. NJ separately covers childless filers aged 18–24, so the 20-year-old head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"Returned $0 without considering the NJ EITC. That credit covers childless filers aged 18+ and pays this head 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"Recognized NJ's age-18 childless rule and estimated a phased-in federal-equivalent credit of about $309 on $4,033 of net earnings, then took 40% to get $123. For this age-only eligibility pathway, the engine pays 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Recognized NJ's age 18–24 childless extension and took 40% of a phased-in federal-equivalent credit of about $307 to get $123. For this age-only eligibility pathway, the engine pays 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"Stated $0 without considering the NJ EITC. That credit covers childless filers aged 18+, and the 20-year-old head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Tied NJ refundable credits to federal EITC eligibility and qualifying children. NJ extends its EITC to childless filers aged 18+ who fail the federal age test, so the head qualifies."
-us,scenario_056,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"Used a minimum age of 21 for NJ refundable credits. NJ's childless EITC starts at age 18, so the 20-year-old head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"Required federal EITC eligibility for the NJ EITC. NJ extends its credit to childless filers aged 18–24 who fail the federal age-25 test, so the head qualifies."
-us,scenario_056,state_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"Denied the NJ EITC because income falls below the filing threshold. Filing is assumed, the EITC is refundable without any liability, and NJ covers childless filers aged 18+, so the head qualifies."
-us,scenario_056,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"Applied the federal childless age range of 25–64 to New Jersey. NJ extends its EITC to childless filers aged 18 and older, so the 20-year-old head qualifies."
-us,scenario_056,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"Used a minimum age of 21 for the NJ childless EITC. That threshold is outdated; NJ lowered it to 18, so the 20-year-old head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Found no NJ refundable credit and missed the NJ EITC for childless filers aged 18+. For this age-only eligibility pathway, the engine pays 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"Required children or other explicit indicators for NJ refundable credits. The NJ EITC covers childless filers aged 18+, and the head's $4,033.39 of net SE earnings qualifies."
-us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"Made NJ refundable credits depend on dependents and positive NJ liability. The NJ EITC is refundable regardless of liability and covers childless filers aged 18+, so the head qualifies."
-us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Asserted that no state EITC applies. NJ extends its EITC to childless filers aged 18+, so the 20-year-old head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"Claimed the $50 NJ property-tax credit as refundable. For filers below the NJ filing threshold, this refundable credit requires being 65+, blind, or disabled, and the model missed the NJ EITC for childless filers aged 18+."
-us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Computed the federal-formula credit as $332, which is 7.65% of gross SE income of $4,340, without first subtracting half the SE tax to reach $4,033.39 of earned income. It then multiplied by 40%, giving $132.80. For this age-only eligibility pathway, the engine pays 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"Recognized NJ's age-18 childless rule and calculated a federal-equivalent phase-in on earnings net of half SE tax ($308.55), then took 40% to get $123.42. For this age-only eligibility pathway, the engine pays 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"Concluded that the head fails the state EITC requirements. NJ extends its EITC to childless filers aged 18+, so the 20-year-old head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"Recognized NJ's childless extension to a 20-year-old and calculated the $308.55 federal-equivalent phase-in, then took 40% to get $123.42. For this age-only eligibility pathway, the engine pays 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Recognized NJ's age-18 childless EITC rule and calculated a federal-equivalent phase-in on net earned income, then took 40% of the $308.55 phased-in credit to get $123.42. For this age-only eligibility pathway, the engine pays 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,"Found no NJ refundable credit for a childless 20-year-old. NJ's EITC covers childless filers aged 18+ and pays this head 40% of the $664 federal childless maximum, or $265.60."
-us,scenario_056,state_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"Returned $0 and missed the NJ EITC for childless filers aged 18+. That credit applies to the head's $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"Derived the NJ EITC only from a $0 federal EITC. NJ computes a federal-equivalent credit for childless filers aged 18–24 who fail the federal age test, so the head qualifies."
-us,scenario_056,state_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"Assumed no federal EITC means no NJ EITC. NJ separately extends its EITC to childless filers aged 18+, so the 20-year-old head qualifies."
-us,scenario_056,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,"Correctly excluded the $50 property-tax credit but tied the NJ EITC to the $0 federal EITC. NJ extends its EITC to childless filers aged 18+, so the head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"Set the NJ EITC at 40% of a $0 federal EITC. NJ covers childless filers aged 18–24 who fail the federal age test with a federal-equivalent credit, so the head qualifies."
-us,scenario_056,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"Treated federal EITC ineligibility as disqualifying for the NJ EITC. NJ extends its credit to childless filers aged 18+, so the 20-year-old head qualifies."
-us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value or explanation for state_refundable_credits, so there was no answer to score."
-us,scenario_056,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"Applied the federal 25–64 childless age range to the NJ EITC. NJ extends its EITC to childless filers aged 18+, so the 20-year-old head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Treated household composition with no children as ruling out NJ credits. NJ's EITC covers childless filers aged 18+, so the head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,muse-spark-1.3,llm_error,credit_phaseout,False,"Required qualifying children for NJ refundable credits. NJ's EITC covers childless filers aged 18+, so the 20-year-old head qualifies."
-us,scenario_056,state_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"Required federal EITC eligibility as the basis for NJ credits. NJ extends its EITC to childless filers aged 18–24 who fail the federal age test, so the head qualifies."
-us,scenario_056,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Computed the NJ EITC as a share of a $0 federal EITC for an age-ineligible filer. NJ separately covers childless filers aged 18+, so the head qualifies based on $4,033.39 of net SE earnings."
-us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"Required qualifying children for the NJ EITC. NJ's EITC covers childless filers aged 18+, and the head's $4,033.39 of net SE earnings qualifies."
-us,scenario_056,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"Required qualifying children and NJ tax liability for refundable credits. The NJ EITC is refundable regardless of liability and covers childless filers aged 18+, so the head qualifies."
+us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"Applied the federal minimum age of 25 for childless workers to the New Jersey EITC. It missed that New Jersey pays its 40% EITC to childless filers aged 18 and older, so this 20-year-old with $4,033.39 of net earnings qualifies."
+us,scenario_056,state_refundable_credits,claude-fable-5.1,llm_error,credit_phaseout,False,"Applied New Jersey's age-18 childless EITC expansion and computed 40% × (7.65% × $4,033.39 = $308.55) = $123.42. The $265.60 target equals 40% of the full $664 maximum childless credit, so the entire $142.18 gap comes from the model capping the credit at the phased-in $308.55."
+us,scenario_056,state_refundable_credits,claude-fable-5.2,llm_error,credit_phaseout,False,"Reduced $4,340 by half of SE tax to get $4,033.39, phased in at 7.65% to $308.55, and took 40% to get $123.42. The $265.60 target is 40% of the $664 maximum childless credit, so the gap comes entirely from the model capping the credit at the phased-in amount."
+us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"Claimed New Jersey has no state EITC or other refundable credits. In fact, New Jersey has a refundable EITC worth 40% of the federal-formula credit, and it covers childless filers aged 18 and older, which includes this 20-year-old with self-employment earnings."
+us,scenario_056,state_refundable_credits,claude-haiku-5.5,llm_error,credit_phaseout,False,"Concluded that no credit applies because there is no New Jersey tax liability and no children. The New Jersey EITC is refundable regardless of liability and covers childless filers aged 18 and older, so the head's $4,033.39 of net self-employment earnings produces a credit."
+us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"Used an outdated minimum age of 21 for New Jersey's childless EITC. New Jersey now pays the credit to childless filers aged 18 and older, so the 20-year-old head qualifies on net self-employment earnings of $4,033.39."
+us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"Treated unmarried-partner status, the lack of children, and unearned income as disqualifying. None of these bars the New Jersey EITC: alimony is not investment income, and New Jersey covers childless filers aged 18 and older, so the $4,033.39 of net self-employment earnings qualifies."
+us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"Required federal EITC eligibility, with its age-25 minimum for childless workers. It missed that New Jersey extends its 40% EITC to childless filers aged 18 and older who fail only the federal age test."
+us,scenario_056,state_refundable_credits,claude-opus-5.5,llm_error,credit_phaseout,False,"Applied the age-18 expansion and computed 40% × 7.65% × $4,033.39 = $123.42. The $265.60 target is 40% of the $664 maximum childless credit, so the gap comes entirely from the model capping the credit at the phased-in $308.55."
+us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"Stated that New Jersey's EITC requires federal eligibility, including the age 25–64 window for childless workers. New Jersey extends the credit to childless filers aged 18 and older, so the 20-year-old head qualifies."
+us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Applied the federal childless minimum age of 25 and speculated about dependent status that the facts do not support. New Jersey's EITC covers childless filers aged 18 and older, so the head's net self-employment earnings of $4,033.39 generate a credit."
+us,scenario_056,state_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"Claimed New Jersey's $50 property tax credit, which is refundable only for filers who are 65 or older, blind, or disabled. It also overlooked the New Jersey EITC, which is available to childless filers aged 18 and older and is the actual refundable credit here."
+us,scenario_056,state_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"Required federal EITC eligibility, which a childless 20-year-old fails on age. New Jersey extends its 40% EITC to childless filers aged 18 and older, so the head qualifies on $4,033.39 of net earnings."
+us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"Set the New Jersey EITC to 40% of a $0 federal EITC. It missed New Jersey's separate eligibility for childless filers aged 18 and older, which computes the credit on the federal formula even when the federal age test fails."
+us,scenario_056,state_refundable_credits,deepseek-v4-pro-0813,llm_error,credit_phaseout,False,"Applied the age-18 expansion but phased in on gross self-employment income of $4,340. Earned income must first be reduced by the deductible half of SE tax ($306.61) to $4,033.39. The $265.60 target equals 40% of the $664 maximum childless credit, not any phase-in amount."
+us,scenario_056,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,Set the New Jersey EITC to 40% of a $0 federal EITC because the head is 20 and has no children. It missed New Jersey's coverage of childless filers aged 18 and older.
+us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"Made New Jersey EITC eligibility depend on federal EITC eligibility. New Jersey extends its credit to childless filers aged 18 and older who fail only the federal age-25 test, as this 20-year-old does."
+us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"Gave no derivation. The $0 answer shows it never applied New Jersey's childless EITC for filers aged 18 and older to the head's $4,033.39 of net self-employment earnings."
+us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"Applied the age-18 expansion and took 40% of a phased-in credit of about $309, giving $123. The $265.60 target is 40% of the $664 maximum childless credit, so the gap comes entirely from capping the credit at the phase-in amount."
+us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"Applied the 40% match for 18–24-year-olds to a phased-in federal-formula credit of about $307 on roughly $4,008 of earnings, giving $123. The $265.60 target is 40% of the $664 maximum childless credit, which no phase-in computation on these earnings produces."
+us,scenario_056,state_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"Gave no substantive reasoning. The $0 answer shows it never recognized New Jersey's EITC for childless filers aged 18 and older, which applies to the head's $4,033.39 of net earnings."
+us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Tied the New Jersey credit to $0 federal EITC eligibility. New Jersey extends its EITC to childless filers aged 18 and older, so this 20-year-old with self-employment earnings qualifies."
+us,scenario_056,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"Used an outdated minimum age of 21 for New Jersey's childless EITC. The current threshold is 18, so the 20-year-old head qualifies."
+us,scenario_056,state_refundable_credits,gemini-3.8-flash,llm_error,credit_phaseout,False,"Required federal EITC eligibility or a child under 6, mixing in the New Jersey Child Tax Credit test. It missed the New Jersey EITC for childless filers aged 18 and older, which covers this head."
+us,scenario_056,state_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"Applied a 65-or-older, blind, or disabled test, which governs the refundable property tax credit, and treated it as also ruling out the New Jersey EITC. The EITC has no such requirement and covers childless filers aged 18 and older with earned income."
+us,scenario_056,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"Carried the federal age 25–64 window for childless workers over to the New Jersey EITC. New Jersey covers childless filers aged 18 and older, so the head qualifies."
+us,scenario_056,state_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"Used a 21-and-older minimum age for New Jersey's childless EITC. The current threshold is 18, which makes the 20-year-old head eligible."
+us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Gave no rule-based reasoning. The $0 answer omits New Jersey's EITC for childless filers aged 18 and older, which applies to the head's $4,033.39 of net self-employment earnings."
+us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"Treated the absence of children as ruling out New Jersey refundable credits. The New Jersey EITC is available to childless filers aged 18 and older with earned income, as this head has."
+us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,Required dependents or positive New Jersey tax liability for a refundable credit. The New Jersey EITC is refundable without liability and covers childless filers aged 18 and older.
+us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"Asserted without analysis that no New Jersey EITC applies. It missed that New Jersey extends the credit to childless filers aged 18 and older, which includes this 20-year-old with net self-employment earnings."
+us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"Awarded the $50 property tax credit, which is refundable only for filers who are 65 or older, blind, or disabled. It also skipped the New Jersey EITC for childless filers aged 18 and older, which is the refundable credit that applies."
+us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Applied the age-18 expansion but computed the federal-formula credit as $332 (7.65% × $4,340 gross), without first subtracting the $306.61 half-SE-tax deduction from earned income. The $265.60 target is 40% of the $664 maximum childless credit."
+us,scenario_056,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"Applied New Jersey's age-18 extension and a 40% match to the phased-in credit on $4,033.39 of earnings, giving $123.42. The $265.60 target equals 40% of the $664 maximum childless credit, so the gap comes entirely from capping at the phased-in $308.55."
+us,scenario_056,state_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"Stated that the head does not qualify for the New Jersey EITC. New Jersey covers childless filers aged 18 and older with earned income, so the 20-year-old with $4,033.39 of net self-employment earnings qualifies."
+us,scenario_056,state_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"Extended the New Jersey EITC to the childless 20-year-old and took 40% of the phased-in $308.55, giving $123.42. The $265.60 target is 40% of the $664 maximum childless credit, so the gap comes entirely from capping at the phase-in amount."
+us,scenario_056,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Applied the age-18 childless expansion and a 40% match to the phased-in credit on net earnings, giving $123.42. The $265.60 target is 40% of the $664 maximum childless credit rather than of the $308.55 phase-in amount."
+us,scenario_056,state_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,Assigned no credit to a childless 20-year-old. It missed that New Jersey pays its EITC to childless filers aged 18 and older with earned income.
+us,scenario_056,state_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"Gave no derivation. The $0 answer omits New Jersey's EITC for childless filers aged 18 and older, which applies to the head's net self-employment earnings of $4,033.39."
+us,scenario_056,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,Derived the New Jersey EITC solely from a $0 federal EITC. It missed New Jersey's separate eligibility for childless filers aged 18–24 who fail only the federal age test.
+us,scenario_056,state_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,Assumed that no federal EITC means no New Jersey EITC. New Jersey computes its credit on the federal formula for childless filers aged 18 and older even when they are federally ineligible on age.
+us,scenario_056,state_refundable_credits,grok-4.7,llm_error,credit_phaseout,False,Correctly excluded the $50 property tax credit but derived the New Jersey EITC as a share of a $0 federal EITC. It missed New Jersey's extension of the credit to childless filers aged 18 and older.
+us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"Took 40% of a $0 federal EITC. It ignored that New Jersey grants its EITC to childless filers aged 18 and older, computed as if the federal age test were met."
+us,scenario_056,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"Made the New Jersey EITC depend on federal eligibility. It missed New Jersey's age-18 threshold for childless filers, which makes the 20-year-old head eligible."
+us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value or explanation for state_refundable_credits, so there was no parseable answer to score."
+us,scenario_056,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,Applied the federal age 25–64 window for childless workers to the New Jersey EITC. New Jersey extends the credit to childless filers aged 18 and older.
+us,scenario_056,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"Ruled out all New Jersey refundable credits based on household composition. The New Jersey EITC covers a single childless filer aged 18 or older with earned income, as here."
+us,scenario_056,state_refundable_credits,muse-spark-1.3,llm_error,credit_phaseout,False,Required qualifying children for a New Jersey refundable credit. The New Jersey EITC is available to childless filers aged 18 and older with net self-employment earnings.
+us,scenario_056,state_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,Used the absence of federal EITC eligibility as the basis for $0. New Jersey's EITC covers childless filers aged 18 and older who fail only the federal age-25 test.
+us,scenario_056,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"Treated the New Jersey EITC as a percentage of a $0 federal EITC for an age-ineligible childless filer. It missed New Jersey's age-18 extension, which computes the credit for exactly this filer."
+us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"Required qualifying children for New Jersey EITC eligibility. New Jersey pays the credit to childless filers aged 18 and older with earned income, which covers this head."
+us,scenario_056,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,Required qualifying children and New Jersey tax liability. The New Jersey EITC is refundable and covers childless filers aged 18 and older with earned income.
us,scenario_056,tanf,glm-5.3,llm_error,categorical_eligibility,False,"The model read the $12,000 `financial_assistance` input as if it were reported TANF receipt and echoed it back verbatim as the computed benefit, when the question asks what PolicyEngine computes for `tanf` from the household facts. It never applied TANF's categorical requirement that the assistance unit contain a dependent child — New Jersey's WorkFirst NJ TANF covers only units with a child under 18 (or 18 and in school full time), while childless adults fall to WFNJ General Assistance, which is not TANF — so this childless 20-year-old head yields $0. Its answer of exactly 12000 is arithmetically identical to the input line it copied, confirming no eligibility test or benefit computation was performed at all."
us,scenario_056,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no `tanf` entry in its `outputs` object and no explanation, so the required key was absent rather than substantively wrong. The correct derivation is that WorkFirst NJ TANF requires an assistance unit containing a dependent child, and this single 20-year-old head with zero children fails that categorical test, producing $0."
us,scenario_057,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It correctly put the couple on the EITC plateau, but it used a max of about $659 and then cut that to $636 for 'lookup-table rounding'. On the plateau the credit is the full 2026 childless maximum of $664, with no rounding reduction."
@@ -5376,57 +5438,58 @@ us,scenario_057,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_out
us,scenario_057,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model computed the correct $1,469.46 from $9,604.40 × 15.3%, then added roughly $82.60 of 0.9% Additional Medicare Tax by treating $400 as the AMT threshold; the AMT threshold is $250,000 of combined earnings for joint filers, and the prompt explicitly excludes Additional Medicare Tax from this output. Its arithmetic is also self-contradictory — the stated components sum to $1,552.06, not the $1,531.79 submitted — where the correct answer is the unadjusted $1,469.47."
us,scenario_057,self_employment_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model fabricated a ""$12,000 threshold for self-employment tax liability"" and zeroed the tax; no such filing floor exists, and the only threshold in §1402(b) is $400 of net earnings, which $9,604.40 clears. Applying the 15.3% rate to that base gives $1,469.47."
us,scenario_057,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model derived the correct liability — 0.153 × $9,604.40 = $1,469.47 — and then reported half of it, confusing the §164(f) above-the-line deduction for one-half of SE tax with the tax owed; that deduction reduces AGI and never reduces the §1401 liability itself. Its submitted $734.76 is not even the correct half ($734.74), and the requested output is the full $1,469.47."
-us,scenario_057,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"It counted only the $10,400 of self-employment income and $38 of disability benefits (~$870/month) and never added the head's SSI of $602.58/month, which is countable unearned income for SNAP. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"It counted couple-rate SSI of ~$1,099/month, but the reference SNAP income includes only the head's individual-rate SSI of $602.58/month ($994 FBR minus $391.42 countable income). That made net income $1,586 instead of $1,090.08 and cut the benefit from $219 to $70/month. Using FY2026 parameters for October–December follows the publication convention."
-us,scenario_057,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,It guessed that income exceeded the net limit and returned $0. The engine counts the full $866.67/month of self-employment income. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,It asserted with no computation that the asset and income tests deny SNAP. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"It left out the head's $602.58/month SSI and subtracted a monthly $219 standard deduction from annual income. Then it abandoned its own $3,990 result and output $6,240 ($10,400 × 0.6), which is not derived from any benefit formula. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"It omitted the head's $602.58/month SSI, treated the $38 in annual disability benefits as $456 a year, and then raised its own $4,536 estimate to $5,736 for medical deductions that don't exist ($40/year of OTC costs is far below the $35/month threshold). The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"It treated net income as near zero, ignoring the $866.67/month of self-employment earnings and the head's $602.58 SSI, and used a two-person maximum of ~$304 instead of $546. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"It counted couple-rate SSI of $1,098.58/month, but the reference counts only the head's individual-rate SSI of $602.58/month, with none for the spouse. The recorded alternative SSI reading yields $70/month, or $840 annually. Using FY2026 parameters through December follows the publication convention. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"It never counted the head's $602.58/month SSI, and it applied a 40% cost-of-production reduction to self-employment income that the reference does not apply; the full $866.67/month is counted. It also used a $535 maximum. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"It reduced self-employment income by an SE-tax-based factor, ignored the head's $602.58/month SSI, invented a medical deduction from $40/year of OTC costs, and gave a net income (~$2,000/month) that doesn't match its own inputs. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"It included couple-rate SSI of ~$1,099/month, while the reference counts only the head's individual-rate SSI of $602.58/month. That pushed net income to ~$1,586 instead of $1,090.08 and the benefit to ~$70 instead of $219/month. Using FY2026 parameters for October–December follows the publication convention."
-us,scenario_057,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Its $869.84 gross income left out the head's $602.58/month SSI, which SNAP counts as unearned income. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,It counted only $870/month of earnings and disability benefits and omitted the head's $602.58/month SSI. It also used the old $204 standard deduction and a $536 maximum instead of $209 and $546. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"It counted $1,067.17/month of SSI, read $38 of annual disability benefits as $38/month, and used a $204 standard deduction and $536 maximum. The held FY2026 amounts are $209 and $546. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Its $869.83 gross income omitted the head's $602.58/month SSI, and it used a $204 standard deduction instead of $209. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"It counted $13,122.96/year of SSI, used a $198/month standard deduction, and claimed a medical deduction from $40/year of OTC costs. The held standard deduction is $209/month, and those OTC costs produce no medical deduction. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,It declared the household ineligible. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"It counted couple-rate SSI and got ~$76/month. The reference counts only the head's individual SSI of $602.58/month, so net income is $1,090.08 and the benefit is $219/month."
-us,scenario_057,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"It counted $12,729/year of SSI plus disability benefits, which assumes couple-rate SSI. The reference counts only the head's $602.58/month (about $7,231/year) individual SSI, which gives net income of $1,090.08 and $219/month instead of its ~$65."
-us,scenario_057,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,It returned zero with no computation. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,Its $268/month implies net income of about $927/month. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,Its $282/month implies net income of about $880/month. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,It counted only $869.83/month of self-employment income and disability benefits and left out the head's $602.58/month SSI. It also used a $200 standard deduction instead of $209. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,gemini-4-argon,prompt_ambiguity,age_disability,False,"It counted $13,048/year of couple-rate SSI, but the reference counts only the head's individual-rate SSI of $602.58/month. Under the head-only SSI reading and held FY2026 schedule, the current engine yields $219/month throughout 2026, or $2,628 annually. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,glm-5.2,prompt_ambiguity,age_disability,False,"It treated SNAP as an unlisted benefit receipt and set it to 0, even though the prompt says to assume take-up and compute program benefits. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,glm-5.3,prompt_ambiguity,age_disability,False,"It counted about $1,100/month of SSI and used a standard deduction of about $230. The held FY2026 standard deduction is $209/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,It claimed counted income was too high for SNAP. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"It set SNAP to 0 because no receipt amount was listed, ignoring the instruction to assume take-up and compute eligibility. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Its $10,438 gross income omits the head's SSI of $602.58/month (about $7,231/year), which SNAP counts as unearned income. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"It left the head's $602.58/month SSI out of income and claimed a medical deduction, even though $40/year of OTC costs is below the $35/month threshold. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"It reached a $70/month benefit by counting couple-rate SSI of about $1,099/month. The reference counts only the head's individual SSI of $602.58/month, so net income is $1,090.08 and the benefit is $219/month."
-us,scenario_057,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"It counted SSI at the couple level and got $70/month. The reference counts only the head's $602.58/month individual SSI, which gives net income of $1,090.08 and $219/month."
-us,scenario_057,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"It estimated $13,183/year of couple-rate SSI. The reference counts only the head's individual-rate SSI of $602.58/month, so annual net income is $13,081, not $19,033, and the benefit is $219/month for nine months plus $232.74/month for October–December."
-us,scenario_057,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Its ~$487.67 net income excludes the head's $602.58/month SSI, which is countable unearned income for SNAP. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"It counted couple-rate SSI (~$1,099/month) and got ~$70/month. The reference counts only the head's individual SSI of $602.58/month, so net income is $1,090.08 and the benefit is $219/month."
-us,scenario_057,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Its $1,586.08 net income contains couple-rate SSI of $1,098.58/month. The reference counts only the head's individual SSI of $602.58, which gives $1,090.08 net and $219/month instead of $70."
-us,scenario_057,snap,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"It estimated SSI at the couple level and got about $73/month. Counting only the head's individual-rate SSI of $602.58/month gives net income of $1,090.08 and $219/month."
-us,scenario_057,snap,grok-4.3,prompt_ambiguity,age_disability,False,It claimed income and assets exceed SNAP limits. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,grok-4.5,prompt_ambiguity,age_disability,False,"It counted roughly $1,058/month of SSI and used a $204 standard deduction. The held FY2026 standard deduction is $209/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,grok-4.6,prompt_ambiguity,age_disability,False,"It counted about $1,085/month of SSI and used a $204 standard deduction and $536 maximum. The held FY2026 amounts are $209 and $546. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,grok-4.7,prompt_ambiguity,age_disability,False,"It counted $13,123/year of couple-rate SSI. The reference counts only the head's $602.58/month individual SSI, so annual net income is $13,081, not $18,973, and the benefit is $219/month."
-us,scenario_057,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,It subtracted a monthly $209 standard deduction from annual income and omitted the head's $602.58/month SSI entirely. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,inkling,prompt_ambiguity,age_disability,False,Its ~$870/month gross income leaves out the head's $602.58/month SSI. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no SNAP value and no explanation, so there was nothing to score."
-us,scenario_057,snap,kimi-k3,prompt_ambiguity,age_disability,False,"It counted $13,183/year of couple-rate SSI, which gave net income of $1,586.08/month. The reference counts only the head's $602.58/month individual SSI, giving $1,090.08 net and $219/month instead of ~$70."
-us,scenario_057,snap,minimax-m3,prompt_ambiguity,age_disability,False,"It said the $912 in bank assets exceeds the asset limit even though $912 is below the ~$4,500 limit it cited itself. The household passes the asset test. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,muse-spark-1.3,prompt_ambiguity,age_disability,False,"It counted only ~$870/month of self-employment income and disability benefits and left out the head's $602.58/month SSI, and it used a $536 maximum. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,muse-wonder,prompt_ambiguity,age_disability,False,"It counted about $1,102/month of SSI and estimated a benefit of about $69/month. The recorded alternative SSI reading yields $70/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,ox-alpha,prompt_ambiguity,age_disability,False,Its gross income of $867 earned plus $3 unearned omits the head's $602.58/month SSI. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
-us,scenario_057,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"It excluded all household income on a made-up rule that SSI-eligible people's income doesn't count for SNAP, and it used a $635 maximum instead of $546. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated."
-us,scenario_057,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,It concluded the household gets no SNAP without any computation. The head-only SSI reading yields $219/month. This output is excluded from scoring because the spouse's SSI disability criterion is unstated.
+us,scenario_057,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Left the household's SSI out of SNAP gross income, using only about $870/mo of self-employment and disability income. The reference adds $602.58/mo of SSI, which raises net income to $1,090.08 and lowers the benefit to $219/mo. Its final $380/mo was also a hand adjustment that does not follow from its own $391 calculation."
+us,scenario_057,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Counted SSI at the eligible-couple rate of about $1,099/mo, while the reference counts only $602.58/mo of SSI (the head's individual payment). That made its net income $1,586 instead of $1,090.08 and its benefit $70/mo instead of $219/mo. It also used the $546 maximum all year instead of $558.24 for Oct–Dec."
+us,scenario_057,snap,claude-fable-5.2,prompt_ambiguity,age_disability,False,"Counted $13,202/yr of couple-rate SSI. The reference counts $602.58/mo ($7,231/yr), so its annual net income of $19,014 is too high against the reference's $1,090.08/mo. It also used a flat $6,552 annual maximum, missing the higher Oct–Dec allotment of $558.24."
+us,scenario_057,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Concluded the benefit was $0 without any computation, citing net income over the limit, which is wrong. Net income of $1,090.08 is 62% of the $1,762.50 poverty line, and $546 − $327 gives $219/mo. It also wrongly cut self-employment income by a self-employment-tax factor, which SNAP does not do."
+us,scenario_057,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Said the asset and net-income tests eliminate the benefit. In fact $912 in assets is far below the limit, and net income of $1,090.08 passes the 100% poverty-line test, so the household gets $219/mo (Jan–Sep) and $232.74/mo (Oct–Dec)."
+us,scenario_057,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Left the household's SSI ($602.58/mo) out of countable income. It also subtracted a monthly standard deduction from annual income, then dropped its own $3,990 result and submitted a near-maximum $6,240 with no supporting math."
+us,scenario_057,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Left the $602.58/mo of SSI out of gross income and introduced an unexplained $456 of unearned income. It reached about $4,536 and then raised it arbitrarily to $5,736 for unspecified disability or medical deductions; the $40/yr of over-the-counter costs is far below the $35/mo threshold."
+us,scenario_057,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Treated net income as near zero and used a two-person maximum of about $304/mo. The actual maximum is $546, and net income is $1,090.08 once self-employment earnings and $602.58/mo of SSI are counted, which gives $219/mo."
+us,scenario_057,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Counted $1,098.58/mo of eligible-couple SSI, while the reference counts only $602.58/mo for the head. That pushed net income to $1,586 instead of $1,090.08 and cut the benefit to $70/mo instead of $219/mo. It also used the $546 maximum for Oct–Dec instead of $558.24."
+us,scenario_057,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,Left SSI ($602.58/mo) out of gross income. It also applied a 40% cost-of-doing-business cut to self-employment income that the reference does not apply; the reference counts the full $866.67/mo before the 20% earned-income deduction. Its $535 maximum is also below the actual $546.
+us,scenario_057,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Applied self-employment-tax adjustments that SNAP does not use and left out the $602.58/mo of SSI. Its claim that net income is about $2,000/mo contradicts its own inputs, and its $297/mo result is not derived from any formula; the correct path gives net income of $1,090.08 and $219/mo."
+us,scenario_057,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Counted about $1,099/mo of couple-rate SSI instead of the reference's $602.58/mo for the head only. That made net income $1,586 instead of $1,090.08 and the benefit about $70/mo instead of $219/mo."
+us,scenario_057,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Left the household's $602.58/mo of SSI out of gross income, leaving only about $870/mo and net income of $488.51 instead of $1,090.08. It also used a $208 standard deduction and a $536 maximum instead of $209 and $546."
+us,scenario_057,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Left SSI ($602.58/mo) out of gross income, which made net income $489.34 instead of $1,090.08. It used outdated parameters too: a $204 standard deduction and a $536 maximum instead of $209 and $546."
+us,scenario_057,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted $1,067.17/mo of couple-level SSI instead of the reference's $602.58/mo for the head. It also counted the $38 of annual disability benefits as a monthly amount and used a $204 standard deduction and $536 maximum, which pushed net income to $1,594.50 instead of $1,090.08."
+us,scenario_057,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Left the $602.58/mo of SSI out of gross income, so net income came to $492.50 instead of $1,090.08. It also used a $204 standard deduction instead of $209, which produced about $398/mo instead of $219."
+us,scenario_057,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Counted $13,122.96/yr of couple-rate SSI instead of the reference's $602.58/mo for the head. It also added a $60 excess medical deduction that does not exist: $40/yr of over-the-counter costs is far below the $35/mo threshold."
+us,scenario_057,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Declared the household ineligible with no computation. It passes every test: gross income is 84% and net income 62% of the poverty line, assets are $912, and it is categorically eligible through SSI. That gives $219/mo for Jan–Sep and $232.74/mo for Oct–Dec."
+us,scenario_057,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Counted couple-level SSI (about $1,090/mo) instead of the reference's $602.58/mo for the head, which drove the benefit down to about $76/mo instead of $219/mo."
+us,scenario_057,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Counted $12,729/yr of SSI and disability income (about $1,061/mo of SSI at the couple level). The reference counts $602.58/mo of SSI, so net income is $1,090.08 and the benefit $219/mo, not about $65/mo."
+us,scenario_057,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Asserted a zero benefit under Louisiana rules with no computation. The household's net income of $1,090.08 and assets of $912 meet the tests, and $546 − $327 gives $219/mo."
+us,scenario_057,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Gave no computation. Its $268/mo implies net income of about $927, which matches neither the version without SSI (about $490) nor the reference's $1,090.08 (self-employment $866.67 + disability $3.17 + SSI $602.58, minus $173.33 and $209). The reference yields $219/mo for Jan–Sep and $232.74 for Oct–Dec."
+us,scenario_057,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Gave no computation. Its $282/mo implies net income of about $880 instead of $1,090.08, which comes from counting $602.58/mo of SSI with self-employment and disability income and taking the 20% earned-income and $209 standard deductions."
+us,scenario_057,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Left the $602.58/mo of SSI out of gross income and used a $200 standard deduction instead of $209. That made net income $496.50 instead of $1,090.08 and the benefit about $397/mo instead of $219/mo."
+us,scenario_057,snap,gemini-4-argon,prompt_ambiguity,age_disability,False,"Counted $13,048/yr of couple-rate SSI instead of the reference's $602.58/mo ($7,231/yr) for the head. That produced annual net income of $18,900, and its flat $6,552 annual maximum misses the $558.24 Oct–Dec allotment."
+us,scenario_057,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Treated SNAP as an unlisted input and set it to zero. The prompt asks for SNAP to be computed with take-up assumed, and the computation gives $219/mo for Jan–Sep and $232.74/mo for Oct–Dec."
+us,scenario_057,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Counted about $1,100/mo of couple-rate SSI instead of the reference's $602.58/mo for the head. It also used a $230 standard deduction instead of $209, which made net income $1,566 instead of $1,090.08."
+us,scenario_057,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Claimed counted income is too high for SNAP. Gross income of $1,472.42 is 84% of the poverty line (the limit is 130%), and net income of $1,090.08 is 62% (the limit is 100%), so the household receives $219/mo."
+us,scenario_057,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Treated SNAP as something the facts must supply and returned 0, instead of computing the benefit with take-up assumed. The computation gives $546 − $327 = $219/mo for Jan–Sep."
+us,scenario_057,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Left the $602.58/mo of SSI out of gross income and counted only $10,438/yr, which gave annual net income of $5,862 instead of $1,090.08/mo. Its $2,496 annual standard deduction ($208/mo) is also slightly below the $209 figure."
+us,scenario_057,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Left SSI out of countable income and claimed a medical deduction, even though $40/yr of over-the-counter costs is far below the $35/mo threshold. Its $340/mo implies net income of about $687 instead of $1,090.08."
+us,scenario_057,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Included couple-rate SSI (about $1,099/mo) in countable income instead of the reference's $602.58/mo for the head, which gave $70/mo instead of $219/mo. It also used the $546 maximum for all 12 months instead of $558.24 for Oct–Dec."
+us,scenario_057,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Counted couple-rate SSI of about $1,099/mo instead of the reference's $602.58/mo for the head, which made net income about $1,586 instead of $1,090.08 and the allotment $70/mo instead of $219/mo."
+us,scenario_057,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Counted about $13,033/yr of eligible-couple SSI instead of the reference's $7,231/yr ($602.58/mo) for the head, which made annual net income $19,033 instead of about $13,081. It also used a flat $6,552 annual maximum."
+us,scenario_057,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Left the $602.58/mo of SSI out of countable income, so net income came to $487.67 instead of $1,090.08 and the benefit to about $400/mo instead of $219/mo."
+us,scenario_057,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Included eligible-couple SSI (about $1,099/mo) in SNAP income instead of the reference's $602.58/mo for the head, which made net income about $1,586 instead of $1,090.08. It also annualized the $546 maximum, skipping the $558.24 Oct–Dec value."
+us,scenario_057,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Estimated SSI at the couple rate, which gave net income of $1,586.08 instead of the reference's $1,090.08 with $602.58/mo of SSI. The result was a $70/mo benefit instead of $219/mo."
+us,scenario_057,snap,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"Counted couple-rate SSI of about $1,090/mo in SNAP income instead of the reference's $602.58/mo for the head, which dropped the benefit to about $73/mo instead of $219/mo."
+us,scenario_057,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Asserted that income and assets exceed SNAP limits. Assets of $912 and net income of $1,090.08 (62% of the poverty line) are within the limits, and the benefit is $219/mo."
+us,scenario_057,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Counted about $1,058/mo of couple-level SSI instead of the reference's $602.58/mo for the head. With a $204 standard deduction, that made net income about $1,551 instead of $1,090.08."
+us,scenario_057,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Counted about $1,085/mo of couple-rate SSI instead of the reference's $602.58/mo, and used a $204 standard deduction and $536 maximum instead of $209 and $546. That made net income about $1,578 and the benefit $63/mo, versus $1,090.08 and $219/mo."
+us,scenario_057,snap,grok-4.7,prompt_ambiguity,age_disability,False,"Counted $13,123/yr of eligible-couple SSI instead of the reference's $602.58/mo for the head, which made annual net income $18,973. It also used a flat $6,552 annual maximum, ignoring the $558.24 Oct–Dec allotment."
+us,scenario_057,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Left the $602.58/mo of SSI out of countable income. It also subtracted the monthly $209 standard deduction once from annual income instead of $209 each month, which produced an inconsistent annual net income of $8,149."
+us,scenario_057,snap,inkling,prompt_ambiguity,age_disability,False,"Left the household's $602.58/mo of SSI out of gross income, which made net income about $488 instead of $1,090.08 and the benefit $399/mo instead of $219/mo."
+us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no SNAP value or explanation, so there was nothing to score. The correct derivation yields $219/mo for Jan–Sep and $232.74/mo for Oct–Dec, or $2,669.22 a year."
+us,scenario_057,snap,kimi-k3,prompt_ambiguity,age_disability,False,"Counted $13,183/yr of eligible-couple SSI instead of the reference's $602.58/mo ($7,231/yr) for the head, which made monthly net income $1,586.08 instead of $1,090.08. It also used the $546 maximum for Oct–Dec instead of $558.24."
+us,scenario_057,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Contradicted itself by saying $912 in bank assets exceeds a roughly $4,500 asset limit. The household passes the asset test easily and receives $219/mo."
+us,scenario_057,snap,muse-spark-1.3,prompt_ambiguity,age_disability,False,"Left the $602.58/mo of SSI out of gross income, so net income came to about $493 instead of $1,090.08. Its $536 maximum is also below the actual $546."
+us,scenario_057,snap,muse-wonder,prompt_ambiguity,age_disability,False,"Counted about $1,102/mo of couple-rate SSI instead of the reference's $602.58/mo for the head, which made net income about $1,590 instead of $1,090.08 and the benefit $69/mo instead of $219/mo."
+us,scenario_057,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Counted only self-employment and disability income (about $870/mo) and left out the $602.58/mo of SSI, so net income came to about $488 instead of $1,090.08 and the benefit to $400/mo instead of $219/mo."
+us,scenario_057,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Wrongly excluded all household income on the theory that SNAP ignores the income of SSI-eligible members. SNAP counts the $866.67 of self-employment earnings, $3.17 of disability benefits and $602.58 of SSI. It also used an inflated $635 maximum instead of $546."
+us,scenario_057,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Concluded there was no positive benefit without any computation. The household is categorically eligible through SSI, passes the gross test at 84% and the net test at 62% of the poverty line, and receives $219/mo for Jan–Sep and $232.74/mo for Oct–Dec."
us,scenario_057,spouse_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Louisiana CHIP as covering low-income adults through a standalone income test. It failed to apply the controlling Medicaid exclusion: the 20-year-old spouse is Medicaid-eligible in the adult category and therefore is not eligible for CHIP, regardless of being below an asserted CHIP income threshold."
us,scenario_057,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model asserted that income exceeds Louisiana's Medicaid limits, but household MAGI is 0.45 x FPL — roughly $10,438 against a 138% FPL expansion cutoff near $29,200 for a household of two in 2026 — so it compared the $10,400 self-employment figure against a threshold far lower than the one the ACA adult expansion actually applies. It also gave the spouse's disability status weight while never running the only test that governs a 20-year-old in an expansion state: MAGI at or below 138% FPL under the ADULT category."
us,scenario_057,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid as requiring an affirmative qualifying attribute in the fact pattern and defaulted to 0 in its absence, missing that Louisiana's ACA adult expansion makes age 19–64 plus MAGI at or below 138% FPL sufficient on its own. The listed facts fully determine eligibility — spouse age 20, non-dependent, household MAGI at 0.45 x FPL — so the ADULT expansion category resolves to eligible without any additional detail."
@@ -5441,57 +5504,58 @@ us,scenario_057,spouse_medicare_eligible,gpt-6.1-sol,llm_error,age_disability,Fa
us,scenario_057,spouse_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model said being disabled satisfies the under-65 Medicare pathway. That pathway requires 24 months of SSDI entitlement (or ESRD/ALS), not disabled status alone. The spouse has no SSDI months, so under-65 eligibility fails."
us,scenario_057,spouse_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented WIC coverage for women up to age 60 and treated disability as a route to postpartum or other categorical eligibility. The spouse is not pregnant, breastfeeding, or postpartum, so satisfying an income limit cannot establish WIC eligibility."
us,scenario_057,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated being a 20-year-old disabled woman as a qualifying WIC category. Age, sex, and disability do not replace the required pregnancy, breastfeeding, or postpartum status, none of which is present."
-us,scenario_057,ssi,claude-fable-5,prompt_ambiguity,age_disability,False,"Submitted a placeholder of 0 without doing any SSI computation. The blind and disabled head is eligible: $994 individual FBR minus $391.42 monthly countable income gives $602.58 a month, or $7,231 a year."
-us,scenario_057,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Treated both spouses as SSI-eligible and applied the $1,491 couple FBR to pooled income. The engine does not classify the 20-year-old spouse as aged, blind, or disabled, so the head is an individual at the $994 FBR. The head's countable income is $391.42 a month, counting only the head's own $14 of unearned income."
-us,scenario_057,ssi,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Read the $14 and $24 annual disability benefits as monthly and left out the one-half earned income exclusion. It then declared the household ineligible even though its own $10,076 was below the couple rate. In the frozen reference, the head's countable income is $391.42 a month against the $994 individual FBR, which gives $7,231 a year."
-us,scenario_057,ssi,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Asserted that the household's earnings rule out SSI without computing countable income. After the $20 general exclusion, the $65 earned exclusion, and the one-half exclusion, the head's countable income is $391.42 a month, well below the $994 individual FBR, so SSI is $7,231."
-us,scenario_057,ssi,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Applied a couple FBR of about $17,604 on the assumption that the 20-year-old spouse qualifies. The engine does not treat the spouse as aged, blind, or disabled, so the head alone gets the $994 individual FBR. It also applied the $20 general exclusion to earnings before unearned income."
-us,scenario_057,ssi,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Used a guessed couple FBR of $1,538 a month and treated the $20 general exclusion as an annual amount. The engine does not treat the spouse as aged, blind, or disabled, so the benefit is the head's $994 individual FBR minus $391.42 monthly countable income."
-us,scenario_057,ssi,claude-opus-5,prompt_ambiguity,age_disability,False,"Claimed that countable self-employment income exceeds the benefit rate. It skipped the $65 earned exclusion and the one-half earned exclusion, which bring the head's countable income down to $391.42 a month, below the $994 individual FBR, for $7,231 a year."
-us,scenario_057,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"The exclusion ordering was correct, but the model pooled both spouses' unearned income and used the $1,491 couple FBR. The engine does not treat the spouse as aged, blind, or disabled, so the head is an individual at $994. The spouse's $2 a month is below the deeming threshold, so the head's countable income is $391.42."
-us,scenario_057,ssi,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Applied a $1,486 a month couple FBR as if the spouse were an eligible disabled spouse. It also netted the $20 general exclusion against the annual unearned total. The frozen reference treats only the head as SSI-eligible at the $994 individual FBR, minus $391.42 monthly countable income."
-us,scenario_057,ssi,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Its reasoning computed a positive $12,590 at a $1,450 couple FBR, yet it submitted 0. Both numbers miss the reference under either reading: the frozen reference gives the head $994 minus $391.42 a month, or $7,231 a year, and the alternative reading gives $13,183. This output is excluded from scoring because the spouse's SSI disability criterion was unlisted."
-us,scenario_057,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Applied the $1,491 couple FBR on the assumption that both spouses are eligible. The engine does not classify the 20-year-old spouse as aged, blind, or disabled, so the head's benefit is the $994 individual FBR minus $391.42, which is $602.58 a month."
-us,scenario_057,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Calculated a separate benefit of about $991 a month for the spouse and overstated the head's monthly countable income at about $430. The frozen reference gives the spouse $0 and uses $391.42 of countable income against the head's $994 FBR. The spouse's SSI disability criterion was unlisted, so this output is excluded from scoring."
-us,scenario_057,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Used an estimated $1,483 couple FBR and pooled income. The frozen reference treats the head as an individual with an ineligible spouse, so the $994 individual FBR applies, minus $391.42 countable income."
-us,scenario_057,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Applied a $1,486 couple FBR and counted $18 of unearned income on top of earnings without carrying the unused general exclusion over to earned income. The engine uses the $994 individual FBR for the head, minus $391.42 countable income."
-us,scenario_057,ssi,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Applied the $1,491 couple FBR to pooled couple income. The frozen reference treats the spouse as SSI-ineligible, so the head is paid at the $994 individual rate, minus $391.42, which is $7,231 a year."
-us,scenario_057,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Used a $1,486 a month couple maximum on the assumption that both spouses qualify. The engine treats only the head as aged, blind, or disabled, so the $994 individual FBR applies."
-us,scenario_057,ssi,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Claimed that earned income disqualifies the household. It skipped the $65 and one-half earned income exclusions, which leave countable income of $391.42 a month, below the $994 FBR."
-us,scenario_057,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Used the couple maximum benefit, which treats the 20-year-old spouse as an eligible spouse. The frozen reference treats only the head as SSI aged, blind, or disabled, so the benefit is $994 minus $391.42 a month."
-us,scenario_057,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Applied a $17,400 couple FBR. The frozen reference treats the spouse as failing the SSI aged, blind, or disabled criterion, so the head's individual FBR of $994 a month applies."
-us,scenario_057,ssi,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Asserted that income and resources exceed SSI limits. The $912 in resources is under the $3,000 resource limit, and after earned income exclusions the head's countable income is $391.42 a month, below the $994 FBR."
-us,scenario_057,ssi,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Computed a couple benefit on the assumption that both spouses qualify. The frozen reference treats only the head as SSI aged, blind, or disabled, so the $994 individual FBR minus $391.42 applies."
-us,scenario_057,ssi,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Treated the household as an eligible couple at the couple FBR. The engine does not classify the spouse as aged, blind, or disabled, so the head gets the $994 individual rate."
-us,scenario_057,ssi,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Declined to compute SSI and submitted 0. The blind and disabled head qualifies for $994 minus $391.42 countable income, which is $602.58 a month."
-us,scenario_057,ssi,gemini-4-argon,prompt_ambiguity,age_disability,False,"Used a $1,479.75 couple FBR and pooled both spouses' unearned income. The frozen reference treats the spouse as SSI-ineligible, so the head's $994 individual FBR applies."
-us,scenario_057,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"Assumed income exceeds the limits and relied on the absence of listed SSI receipt, even though the prompt says to assume take-up. The head's countable income of $391.42 is below the $994 FBR."
-us,scenario_057,ssi,glm-5.3,prompt_ambiguity,age_disability,False,"Applied a couple FBR of about $1,490.56 because it treated the spouse as eligible. The frozen reference uses the $994 monthly individual FBR for the head."
-us,scenario_057,ssi,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Claimed that assets exceed the SSI limit, but $912 is below the $3,000 resource limit. With $391.42 of countable income, the head receives $7,231."
-us,scenario_057,ssi,gpt-5.4-nano,prompt_ambiguity,age_disability,False,Required listed SSI receipt and ignored the prompt's instruction to assume take-up. The blind and disabled head is eligible for $994 minus $391.42 a month.
-us,scenario_057,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"Applied the $1,491 couple FBR on the assumption that both spouses are eligible. The engine uses the head's $994 individual FBR minus $391.42."
-us,scenario_057,ssi,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Applied a couple FBR of about $1,450. The frozen reference treats only the head as SSI aged, blind, or disabled, so the $994 individual rate applies."
-us,scenario_057,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Used the 2026 couple maximum on the assumption that both spouses meet disability criteria. The engine does not treat the spouse as aged, blind, or disabled, so the $994 individual FBR applies."
-us,scenario_057,ssi,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Computed a couple benefit, which treats the spouse as eligible. The engine pays only the head, at the $994 individual rate minus $391.42."
-us,scenario_057,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"Applied the $17,892 annual couple maximum. The frozen reference treats the spouse as SSI-ineligible, so the head's individual FBR of $994 a month applies."
-us,scenario_057,ssi,gpt-6-luna,prompt_ambiguity,age_disability,False,"Used the $1,491 couple FBR on the assumption that both spouses qualify. The engine uses the $994 individual rate for the head."
-us,scenario_057,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"Applied the $1,491 couple rate. The frozen reference treats only the head as SSI aged, blind, or disabled, so the $994 individual FBR applies."
-us,scenario_057,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Used the $17,892 couple maximum. The frozen reference treats the spouse as SSI-ineligible, so the head gets $994 minus $391.42 a month."
-us,scenario_057,ssi,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"Used the federal couple benefit. The engine pays only the head, at the $994 individual rate."
-us,scenario_057,ssi,grok-4.3,prompt_ambiguity,age_disability,False,"Claimed that countable self-employment income exceeds the FBR. After the earned income exclusions, the head's countable income is $391.42 a month, below $994."
-us,scenario_057,ssi,grok-4.5,prompt_ambiguity,age_disability,False,"Used a $1,486 couple FBR. The frozen reference treats the spouse as SSI-ineligible, so the head gets the $994 individual rate."
-us,scenario_057,ssi,grok-4.6,prompt_ambiguity,age_disability,False,"Applied a $17,832 couple FBR and did not carry the unused general exclusion over to earnings. The engine uses the $994 individual FBR minus $391.42."
-us,scenario_057,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"Applied the $1,486 couple rate. The frozen reference treats the spouse as failing the SSI aged, blind, or disabled criterion, so the $994 individual rate applies."
-us,scenario_057,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"Used a $17,835 couple FBR. The engine pays the head as an individual at $994 a month."
-us,scenario_057,ssi,inkling,prompt_ambiguity,age_disability,False,"Used a $1,490 couple rate and understated countable income at about $370. The frozen reference uses the $994 individual FBR and $391.42 countable income."
-us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SSI value or explanation. The expected value is $994 minus $391.42 a month, which is $7,231 a year."
-us,scenario_057,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Applied the $1,491 couple rate. The frozen reference treats only the head as SSI aged, blind, or disabled, so the $994 individual FBR applies."
-us,scenario_057,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"Concluded that SSI is zero from the small size of the disability benefits, without computing anything. The head qualifies for $994 minus $391.42 a month."
-us,scenario_057,ssi,muse-spark-1.3,prompt_ambiguity,age_disability,False,"Used the $1,491 couple FBR. The frozen reference treats the spouse as SSI-ineligible, so the $994 individual rate applies."
-us,scenario_057,ssi,muse-wonder,prompt_ambiguity,age_disability,False,"Used the $1,491 couple FBR. The frozen reference uses the $994 monthly individual FBR for the head."
-us,scenario_057,ssi,ox-alpha,prompt_ambiguity,age_disability,False,"Used the $1,491 couple FBR. The engine applies the $994 individual rate for the head."
-us,scenario_057,ssi,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Invented FBRs between $674 and $1,412 and treated the annual $14 and $24 benefits as monthly. Its final figure contradicts its own arithmetic. The frozen reference uses $994 minus $391.42 a month, or $7,231 a year."
-us,scenario_057,ssi,qwen3.8-max,prompt_ambiguity,age_disability,False,"Asserted that there is no positive benefit without computing countable income. The head's countable income is $391.42, below the $994 FBR, which gives $7,231."
+us,scenario_057,ssi,claude-fable-5,prompt_ambiguity,age_disability,False,"Submitted a placeholder $0 without doing any SSI calculation. The head is SSI-eligible as blind/disabled with $912 in resources, which is under the $2,000 individual limit. Countable income of $391.42/month against the $994 individual FBR gives $602.58/month, or $7,231/year."
+us,scenario_057,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Treated both spouses as an eligible couple and used the $1,491 couple FBR. Under PolicyEngine the 20-year-old spouse is not SSI aged/blind/disabled, so the head is an individual with an ineligible spouse at the $994/month FBR. The spouse's $24 is not deemed, which makes countable income $391.42/month and the benefit $7,231/year."
+us,scenario_057,ssi,claude-fable-5.2,prompt_ambiguity,age_disability,False,"Applied the $1,491 eligible-couple FBR even though the spouse is not SSI aged/blind/disabled. It also ignored the head's $14 of unearned income, which absorbs part of the general exclusion. The correct figure is the $994 individual FBR minus $391.42 of countable income, or $602.58/month ($7,231/year)."
+us,scenario_057,ssi,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Left out the one-half earned income exclusion, treated the annual $14/$24 disability amounts as monthly, and concluded the household was ineligible even though its own $10,076 countable figure was below its FBR. Correctly applying the $20 and $65 exclusions and the half-remainder exclusion gives the head $391.42/month of countable income against a $994 individual FBR, which is $7,231/year."
+us,scenario_057,ssi,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Declared the head ineligible without doing any calculation. The head is blind/disabled, has $912 in resources (under $2,000), and has only $391.42/month of countable income after the general exclusion, the $65 exclusion, and the half-remainder exclusion. That leaves a $602.58/month benefit against the $994 individual FBR."
+us,scenario_057,ssi,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Used a roughly $17,604 eligible-couple FBR on the assumption that the disabled 20-year-old spouse qualifies. In PolicyEngine the spouse is not SSI-ABD, so the head is paid at the $994/month individual rate. It also applied the general exclusion to earnings before unearned income, which is the wrong order. Correct countable income is $391.42/month, which yields $7,231/year."
+us,scenario_057,ssi,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Used an inflated $1,538/month couple FBR for an assumed eligible couple, and its final number does not follow from its own arithmetic. Only the head is SSI aged/blind/disabled, so the $994 individual FBR applies. Subtracting $391.42 of countable income gives $7,231/year."
+us,scenario_057,ssi,claude-opus-5,prompt_ambiguity,age_disability,False,"Claimed that roughly $9,600 of countable self-employment earnings exceeds the benefit rate. It skipped the one-half earned income exclusion and compared an annual figure as if it were monthly. The head's countable income is only $391.42/month, below the $994 individual FBR, which leaves $602.58/month ($7,231/year)."
+us,scenario_057,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Computed the benefit as an eligible couple at the $1,491 FBR and pooled the spouse's $2/month of unearned income. Under PolicyEngine the spouse is not SSI aged/blind/disabled, so the head gets the $994 individual FBR. The spouse's income falls below the deeming threshold, which makes countable income $391.42 and the benefit $602.58/month ($7,231/year)."
+us,scenario_057,ssi,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Used a $1,486/month couple FBR for an assumed eligible couple. In its final pass it also did not carry the unused general exclusion over to earnings. The head alone is SSI-ABD, so the $994 individual FBR minus $391.42 of countable income ($866.67 − $18.83 − $65, halved) gives $7,231/year."
+us,scenario_057,ssi,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Its own working reached about $12,590 using the $1,450 couple FBR, but it submitted $0 anyway. That couple-rate figure is also wrong, because the spouse is not SSI aged/blind/disabled. The head's benefit is the $994 individual FBR minus $391.42 of countable income, or $7,231/year."
+us,scenario_057,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Applied the $1,491 eligible-couple FBR on the assumption that both spouses qualify. PolicyEngine treats only the head as SSI aged/blind/disabled, so the $994 individual rate applies with no deemed income from the spouse. With $391.42/month of countable income, the benefit is $7,231/year."
+us,scenario_057,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Added a nearly full individual SSI benefit for the spouse, who is not SSI aged/blind/disabled and receives $0. It also overstated the head's countable income at about $430 instead of $391.42/month. The head's benefit alone is $994 − $391.42 = $602.58/month, or $7,231/year."
+us,scenario_057,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Subtracted countable income from an estimated $1,483 couple FBR, treating the spouse as an eligible partner. Only the head is SSI-ABD, so the $994 individual FBR applies. The spouse's $2/month is not deemed, which gives $391.42 of countable income and $7,231/year."
+us,scenario_057,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Used a $1,486 couple FBR and treated the $38 of annual disability benefits as $38/month, which overstated countable income at $418.83. The spouse is not SSI aged/blind/disabled, so the head gets the $994 individual FBR. The head's countable income is $391.42/month, giving $7,231/year."
+us,scenario_057,ssi,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Applied the $1,491 eligible-couple FBR. Under PolicyEngine the spouse is not SSI-ABD, so the head is an individual with an ineligible spouse at the $994/month FBR. With $391.42 of countable income, the benefit is $7,231/year."
+us,scenario_057,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Used a $1,486/month couple maximum on the assumption that both disabled spouses qualify. Only the head is SSI aged/blind/disabled, so the $994 individual FBR applies against $391.42/month of countable income, giving $7,231/year."
+us,scenario_057,ssi,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Asserted that the head's earnings disqualify SSI without applying the earned income exclusions. After the $20 and $65 exclusions and the half-remainder exclusion, the head's countable income is $391.42/month. That is well below the $994 individual FBR and leaves $7,231/year."
+us,scenario_057,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Applied the couples maximum benefit on the assumption that the spouse is SSI-eligible. PolicyEngine treats the spouse as not SSI aged/blind/disabled, so the head is paid at the $994 individual FBR minus $391.42 of countable income, or $7,231/year."
+us,scenario_057,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Started from a $17,400 couple FBR, which is the 2025 couple rate, for an assumed eligible couple. Only the head qualifies as SSI-ABD, so the 2026 individual FBR of $994/month minus $391.42 of countable income gives $7,231/year."
+us,scenario_057,ssi,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Claimed that the household's income and resources exceed SSI limits. The head's $912 in resources is under the $2,000 limit, and countable income of $391.42/month is below the $994 individual FBR. That leaves a $602.58/month benefit ($7,231/year)."
+us,scenario_057,ssi,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Computed an eligible-couple benefit on the assumption that both disabled adults qualify. The spouse is not SSI aged/blind/disabled under PolicyEngine, so the head is paid at the $994 individual FBR less $391.42 of countable income, which is $7,231/year."
+us,scenario_057,ssi,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Treated the head and spouse as an eligible couple at the couple FBR. Only the head is SSI-ABD, so the benefit is the $994 individual FBR minus $391.42 of countable income, or $602.58/month ($7,231/year)."
+us,scenario_057,ssi,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Wrongly assumed that SSI is not simulated for this household and returned $0. The blind/disabled head meets the resource test and has $391.42/month of countable income against the $994 individual FBR, which yields $7,231/year."
+us,scenario_057,ssi,gemini-4-argon,prompt_ambiguity,age_disability,False,"Subtracted countable income from a $17,757 couple FBR, treating the spouse as eligible, and pooled the spouse's $24 into the exclusion calculation. The head alone is SSI-ABD at the $994/month individual rate, with $391.42 of countable income, which gives $7,231/year."
+us,scenario_057,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"Assumed that the earnings exceed SSI limits and that the absence of listed SSI receipt means zero. The prompt says to assume take-up, and the head's countable income of $391.42/month is below the $994 individual FBR. That produces $7,231/year."
+us,scenario_057,ssi,glm-5.3,prompt_ambiguity,age_disability,False,"Used a couple FBR of about $1,490.56 on the assumption that both spouses are SSI-eligible. It also ignored the head's $1.17/month of unearned income when applying the general exclusion. The head alone qualifies, so the $994 individual FBR minus $391.42 gives $7,231/year."
+us,scenario_057,ssi,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Claimed that the household's assets exceed the SSI limit and that its income disqualifies it. The head's $912 in resources is under the $2,000 limit, and countable income after exclusions is only $391.42/month against the $994 individual FBR. That yields $7,231/year."
+us,scenario_057,ssi,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Set SSI to $0 because no SSI receipt was listed, ignoring the instruction to assume take-up. The blind/disabled head qualifies with $391.42/month of countable income against the $994 individual FBR, which gives $7,231/year."
+us,scenario_057,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"Used the $1,491 eligible-couple FBR and pooled the spouse's $24 into the general exclusion. Under PolicyEngine the spouse is not SSI aged/blind/disabled, so the head gets the $994 individual rate with $391.42/month of countable income, or $7,231/year."
+us,scenario_057,ssi,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Applied a couple FBR, which works back to the 2025 $1,450 rate, on the assumption that both adults qualify. Only the head is SSI-ABD, so the 2026 $994 individual FBR minus $391.42 of countable income yields $7,231/year."
+us,scenario_057,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Reduced the 2026 couple maximum on the assumption that both spouses meet SSI disability criteria. PolicyEngine treats the spouse as not SSI-ABD, so the head is paid at the $994 individual FBR less $391.42, which is $7,231/year."
+us,scenario_057,ssi,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Computed a couple benefit on the assumption that both adults are SSI-disabled. Only the head qualifies, so the $994 individual FBR applies. The spouse's $24 is not deemed because it falls below the deeming threshold, which gives $391.42 of countable income and $7,231/year."
+us,scenario_057,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"Measured countable income against the $17,892 couple maximum. The spouse is not SSI aged/blind/disabled, so the head is an individual with an ineligible spouse at $994/month. Countable income is $4,697/year, leaving $7,231/year."
+us,scenario_057,ssi,gpt-6-luna,prompt_ambiguity,age_disability,False,"Applied the $1,491 couple FBR with $392.42 of countable income, which includes the spouse's $2/month. Only the head is SSI-ABD, and the spouse's income falls below the deeming threshold. That makes the benefit $994 − $391.42 = $602.58/month, or $7,231/year."
+us,scenario_057,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"Started from the $1,491 couple benefit on the assumption of an eligible couple. The spouse is not SSI aged/blind/disabled, so the head's $994 individual FBR minus $391.42 of countable income gives $7,231/year."
+us,scenario_057,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Used the $17,892 annual couple maximum on the assumption that both disabled spouses qualify financially and categorically. PolicyEngine treats only the head as SSI-ABD, so the $994/month individual rate applies, giving $7,231/year."
+us,scenario_057,ssi,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"Subtracted countable income from the federal couple benefit, treating the spouse as an eligible individual. Only the head is SSI aged/blind/disabled, so the $994 individual FBR less $391.42 of countable income gives $7,231/year."
+us,scenario_057,ssi,grok-4.3,prompt_ambiguity,age_disability,False,"Claimed that countable self-employment income exceeds the couple FBR without applying the one-half earned income exclusion. The head's countable income is $391.42/month, below the $994 individual FBR, which leaves $7,231/year."
+us,scenario_057,ssi,grok-4.5,prompt_ambiguity,age_disability,False,"Applied a projected $1,486 couple FBR on the assumption that both spouses qualify. The spouse is not SSI-ABD, so the head is paid at the $994 individual FBR minus $391.42, which is $602.58/month ($7,231/year)."
+us,scenario_057,ssi,grok-4.6,prompt_ambiguity,age_disability,False,"Used the $17,832 couple FBR and failed to carry the unused general exclusion over to earnings, which put countable income at $4,810. Only the head is SSI aged/blind/disabled. The $994 individual FBR less $4,697/year of countable income gives $7,231/year."
+us,scenario_057,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"Applied the $1,486 couple rate on the premise that both spouses are SSI-disabled. PolicyEngine treats the spouse as not SSI-ABD, so the head's benefit is the $994 individual FBR minus $391.42 of countable income, or $7,231/year."
+us,scenario_057,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"Used an approximately $17,835 couple FBR for an assumed eligible couple. Only the head qualifies, so the $994/month individual FBR minus $391.42/month of countable income gives $7,231/year."
+us,scenario_057,ssi,inkling,prompt_ambiguity,age_disability,False,"Applied a roughly $1,490 couple FBR and understated countable income at about $370. The spouse is not SSI aged/blind/disabled, so the head gets $994 − $391.42 = $602.58/month, or $7,231/year."
+us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SSI value and no explanation, so there was nothing to score. The correct output is the head's individual benefit: $994 − $391.42 = $602.58/month, or $7,231/year."
+us,scenario_057,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Used the $1,491/month couple rate on the assumption that both disabled spouses qualify. Only the head is SSI-ABD, so the $994 individual FBR applies to $391.42/month of countable income, yielding $7,231/year."
+us,scenario_057,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"Inferred from the small disability benefit amounts that the household receives no SSI, without computing eligibility. The blind/disabled head passes the resource test and has $391.42/month of countable income against the $994 individual FBR, giving $7,231/year."
+us,scenario_057,ssi,muse-spark-1.3,prompt_ambiguity,age_disability,False,"Subtracted about $392 of countable income from the $1,491 couple FBR on the assumption that both spouses are eligible. The spouse is not SSI aged/blind/disabled, so the head gets the $994 individual FBR minus $391.42, or $7,231/year."
+us,scenario_057,ssi,muse-wonder,prompt_ambiguity,age_disability,False,"Applied the $1,491 couple FBR and understated countable income at about $389. Only the head is SSI-ABD, so the benefit is $994 − $391.42 = $602.58/month, or $7,231/year."
+us,scenario_057,ssi,ox-alpha,prompt_ambiguity,age_disability,False,"Used the $1,491 couple FBR and ignored the head's own $1.17/month of unearned income, which absorbs part of the $20 general exclusion. The spouse is not SSI-ABD, so the head's $994 individual FBR minus $391.42 of countable income gives $7,231/year."
+us,scenario_057,ssi,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Cycled through invented FBR values ($674–$1,058) instead of the 2026 $994 individual rate and treated the annual $14/$24 disability amounts as monthly. Its submitted $8,472 does not follow from any of its own calculations. With the spouse not SSI-ABD, the head's $994 FBR minus $391.42 of countable income gives $7,231/year."
+us,scenario_057,ssi,qwen3.8-max,prompt_ambiguity,age_disability,False,"Asserted that the household's income and resources produce no benefit, without computing exclusions. The head's $912 in resources is under $2,000, and countable income of $391.42/month is below the $994 individual FBR, which leaves $7,231/year."
us,scenario_057,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied Louisiana's correct post-reform 3% flat rate but subtracted only $9,000 as the ""MFJ standard deduction"" — that is the repealed $4,500-per-filer personal exemption amount, not the deduction. Under Act 11 of the 2024 Third Extraordinary Session, Louisiana's standard deduction for married filing jointly is $25,000 (indexed beginning 2026), which exceeds the household's ~$9,665 federal AGI, so Louisiana taxable income is zero. Its $665 base is arithmetic on the wrong subtraction, and 3% of a base that does not exist produced the $20."
us,scenario_057,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It reconstructed the pre-2025 Louisiana regime on two counts: it subtracted $4,500-per-filer personal exemptions ($9,000) and applied a graduated first-bracket rate, both eliminated effective tax year 2025 when Louisiana moved to a 3% flat tax with a $25,000 MFJ standard deduction. Its cited ""1.75% on the first $12,500"" matches neither the current flat rate nor the former 1.85% first bracket, whose MFJ width was $25,000. The $25,000 standard deduction alone wipes out the ~$9,665 federal AGI, leaving $0 rather than a $665 taxable base."
us,scenario_057,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_income_tax_before_refundable_credits, so the miss is a contract failure rather than a substantive rule error. The required derivation runs $10,400 of SSTB self-employment income to roughly $9,665 federal AGI after the deductible half of self-employment tax, then against Louisiana's $25,000 married-filing-jointly standard deduction under its 3% flat tax, giving $0."
@@ -5676,53 +5740,54 @@ us,scenario_064,dependent2_medicaid_eligible,grok-4.5,prompt_ambiguity,household
us,scenario_064,dependent2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or explanation for dependent2_medicaid_eligible, so its answer is missing rather than wrong on substance. The correct answer is 0: household MAGI of 3.22× FPL exceeds Wisconsin's child limits, and no disability, pregnancy or caretaker pathway applies."
us,scenario_064,dependent2_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_064,dependent2_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its AGI ($114,422), overtime deduction, $510 QBI deduction and $3,200 of credits all match the correct path, and its own figures give about $7,630 − $3,200 ≈ $4,430. It then added about $1,529 of unexplained 'adjustments' to reach $5,959 instead of reporting tax minus credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It added the $14,000 SE partnership income on top of the $19,350 partnership/S-corp income, which counts it twice. It also subtracted the $1,796 domestic production deduction, which was repealed after 2017, and took a $3,310 QBI deduction on $16,550 instead of $510 on the $2,550 net business income. Together these overstate taxable income by about $9,405 ($77,217 vs $67,812.12)."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $13,899 overtime premium on top of wages that already include it, deducted the full $18,235 capital loss instead of the $3,000 limit, and used a $14,600 standard deduction instead of the $32,200 joint amount for 2026. It never took the overtime deduction, and it applied unexplained extra credits to reach its figure."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It gave no computation. Taxable income is $67,812.12 after the $32,200 standard, $510 QBI and $13,899.29 overtime deductions, and the resulting $7,641.46 of tax is larger than the $3,200 of CTC/ODC, so liability is $4,441.46. A $0 answer assumes deductions and credits wipe out the tax, which they do not."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It left out the 2026 deduction for the $13,899 FLSA overtime premium. It also deducted half of an SE tax on the $14,000 partnership income, but that income nets against the $16,800 farm loss so no SE tax arises. On top of that, it cut the nonrefundable CTC to $1,072 even though tax is well above the full $3,200 of credits, and it added an unexplained $1,000."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $13,899 overtime-premium deduction and added $1,978 of SE tax into income tax; SE tax is a separate liability, and here the farm loss eliminates it anyway. It also never subtracted the CTC/ODC it identified, so its $14,959 is higher than its own pre-credit total."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It subtracted the repealed $1,796 domestic production deduction and an SE-tax deduction that doesn't apply (there is no SE tax here), took a $3,900 QBI deduction instead of $510, and omitted the $13,899 overtime deduction. It then added an unexplained 'additional tax' that pushed its answer above its own pre-credit figure, and it used $2,500 of credits instead of $3,200."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It used pre-OBBBA 2025 values: a $30,000 standard deduction instead of $32,200, 2025 brackets, and a $2,000 CTC instead of $2,200. It never took the $13,899 overtime-premium deduction, and it claimed an SE health insurance deduction and an SE-tax deduction even though net SE income is negative once the farm loss is included."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It omitted the $13,899 overtime deduction and subtracted the repealed $1,796 domestic production deduction. It then dropped the CTC as 'mostly refundable' even though tax is high enough to use the full $3,200 nonrefundably, and it inflated its own ~$9,300 tax estimate to $12,500 for Medicare/NIIT effects that don't apply."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its $128.5k AGI counts the $14,000 SE partnership income twice, on top of the $19,350 partnership income. It used a $3.3k QBI deduction instead of $510 and omitted the $13,899 overtime-premium deduction, which overstates taxable income by about $25k ($92.9k vs $67.8k)."
-us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income in addition to the $19,350 partnership income, subtracted the repealed $1,796 domestic production deduction, used a $3,310 QBI deduction and a $30,750 standard deduction, and never took the $13,899 overtime deduction. It also treated the 18-year-old as a CTC-qualifying child ($4,000 CTC); only the 12-year-old qualifies, with the two older dependents getting the $500 ODC each."
-us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the TCJA-sunset rules: a $15,900 standard deduction, personal exemptions, a 15% bracket and a $1,000 CTC. OBBBA made the TCJA structure permanent, so 2026 has a $32,200 joint standard deduction, no exemptions, a 12% bracket, a $2,200 CTC plus $500 ODCs, and the overtime-premium deduction. It also subtracted health premiums from wages, which is wrong."
-us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the expired TCJA-sunset rules: personal exemptions, pre-TCJA brackets, and a $1,000 CTC phasing out at $110,000. Under OBBBA, 2026 has a $32,200 standard deduction, a $13,899.29 overtime deduction, a $2,200 CTC and $500 ODCs with a $400,000 phaseout, so the full $3,200 credit applies."
-us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its $114,422 AGI is right, but it omitted the $13,899 overtime-premium deduction and subtracted the repealed $1,796 domestic production deduction. It also used a $30,750 standard deduction instead of $32,200 and a $2,000 CTC instead of $2,200."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied pre-TCJA rules: personal exemptions and a $1,000 CTC. It also subtracted the employer-sponsored insurance premiums from wages and itemized medical expenses built from those same premiums, which aren't included in wages and aren't deductible a second time. Under 2026 law it should have taken the $32,200 standard deduction, the $510 QBI deduction, the overtime deduction and $3,200 of credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It itemized instead of taking the $32,200 standard deduction and gave no sign of the $13,899.29 overtime deduction or the $3,200 of CTC/ODC. Its $11,090 is more than the correct $7,641.46 pre-credit tax on $67,812.12 of taxable income."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used pre-TCJA rules (personal exemptions, a $16,200 standard deduction, a $1,000 CTC), subtracted ESI premiums and the repealed domestic production deduction from AGI, and omitted the overtime deduction. It also added a $2,000 early-withdrawal penalty, which is not part of income tax before refundable credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA personal exemptions and a $1,000 CTC, and it understated AGI at $94,756 by subtracting employer insurance premiums from wages. The correct 2026 OBBBA path uses $114,421.41 of AGI, the $32,200 standard deduction, the $13,899.29 overtime deduction, the $510 QBI deduction and $3,200 of CTC/ODC."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $11,940 is above even the correct $7,641.46 pre-credit tax, which means it left out the overtime and QBI deductions and the $3,200 of CTC/ODC that reduce tax to $4,441.46."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $6,215 is $1,774 too high, which is about $14,800 of extra taxable income at 12%. That matches leaving out the $13,899.29 FLSA overtime-premium deduction while taking the standard deduction and the $3,200 of CTC/ODC."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $4,324 is $117 too low, which is about $975 of excess deductions at 12%. It overstated a deduction; the correct figures are the $32,200 standard deduction and a $510 QBI deduction (20% of the $2,550 net of partnership income and the farm loss), giving $67,812.12 of taxable income."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its $4,057 is $384 too low, which is about $3,200 of excess deductions at 12%. That matches a QBI deduction of about 20% of $16,550 instead of 20% of the $2,550 net business income ($510), with the overtime deduction and $3,200 of credits otherwise applied."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It never took the $13,899 overtime-premium deduction and deducted half of an SE tax that doesn't exist, since the farm loss offsets the SE partnership income. It used the $2,000 pre-OBBBA CTC instead of $2,200 and added a $2,000 early-withdrawal penalty, which is not part of income tax before refundable credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so there was nothing to score against the $4,441.46 reference."
-us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_income_tax_before_refundable_credits, so there was nothing to score against the $4,441.46 reference."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits wipe out the tax. After the $32,200 standard, $510 QBI and $13,899.29 overtime deductions, taxable income is still $67,812.12, and the $7,641.46 of tax on it exceeds the $3,200 of CTC/ODC, leaving $4,441.46."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits eliminate liability. $114,421.41 of AGI less the standard, QBI and overtime deductions leaves $67,812.12 of taxable income and $7,641.46 of tax, which is more than the $3,200 of nonrefundable CTC/ODC."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $2,219 is $2,222 too low, which is about $18,500 of excess deductions at 12%. The capital loss is limited to $3,000 against ordinary income, and the QBI deduction is only $510, so the correct taxable income is $67,812.12."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its $4,485 is $43.54 too high, which is about $363 too little in deductions at 12%. It understated a small deduction, such as the $510 QBI deduction or the $108.19 IRA deduction, or used a slightly wrong 2026 bracket or standard-deduction figure instead of the $32,200 deduction and $24,800 10% bracket."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Its explanation subtracts only the qualifying-child credit, and $5,455 is about $1,000 above the correct figure. It left out the $500 credit for other dependents for each of the two adult dependents (the disabled 27-year-old and the 18-year-old)."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its $92,978 AGI comes from subtracting the $19,648 employer-sponsored insurance premiums, which aren't part of the reported wages, and the $1,796 domestic production deduction, which was repealed after 2017. This understates AGI relative to the correct $114,421.41 and taxable income by about $21,443."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $5,570 corresponds to taxable income $9,405 too high. That is exactly what you get from adding the $14,000 SE partnership income on top of the $19,350 partnership income, subtracting the repealed $1,796 domestic production deduction, and taking a $3,310 QBI deduction instead of $510."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the repealed $1,796 domestic production deduction, which lowered AGI to $112,626 instead of $114,421.41. It also claimed $1,500 of other-dependent credits; only the two adult dependents qualify, for $1,000."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its $13,500 is well above the correct $7,641.46 pre-credit tax. It ignored the $13,899.29 overtime and $510 QBI deductions that bring taxable income to $67,812.12, and it did not net the $3,200 of CTC/ODC."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Its $114,422 AGI is right, but it applied the TCJA-sunset rules: a $17,000 standard deduction, personal exemptions, a 15% bracket and a $1,000 CTC phasing out at $110,000. OBBBA made the TCJA structure permanent, so it should have used the $32,200 standard deduction, the overtime deduction, the 12% bracket and $3,200 of CTC/ODC."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It added the $14,000 SE partnership income on top of the $19,350 partnership income, which cancels nearly all of the $13,899 overtime deduction it took. It also used a $30,700 standard deduction instead of $32,200 and a $3,310 QBI deduction instead of $510."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income twice and took a $3,310 QBI deduction instead of $510, putting taxable income at $79,013 instead of $67,812.12. It also added a $2,000 early-distribution penalty, which is not part of income tax before refundable credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted $21,208 of employer and other health premiums from AGI, counted the $14,000 SE partnership income twice, deducted half of an SE tax that doesn't exist, used the repealed $1,796 domestic production deduction as QBI, and omitted the overtime deduction. It also claimed only a $2,000 CTC, with no $500 other-dependent credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"It added a $2,000 10% early-withdrawal penalty on the 401(k) distribution, which is not part of income tax before refundable credits. It also understated AGI at $97,723 instead of $114,421.41 and subtracted the repealed $1,796 domestic production deduction in place of the $510 QBI deduction."
-us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It added the $14,000 SE partnership income on top of the $19,350 partnership income and deducted Dependent 1's 401(k) and IRA contributions on the filers' return. It also took a $6,670 QBI deduction instead of $510, omitted the $13,899 overtime deduction, and denied the 18-year-old's $500 ODC while using a $2,000 CTC instead of $2,200."
-us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It did not remove the $2,315.40 traditional 401(k) deferral from wages, which put AGI at $116,737. It also subtracted the repealed $1,796 domestic production deduction instead of the $510 QBI deduction, overstating taxable income by about $1,030."
-us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It gave no computation. The correct path leaves $67,812.12 of taxable income and $7,641.46 of tax, and $3,200 of CTC/ODC brings that to $4,441.46. A $0 answer treats the household as having no remaining liability, which is wrong."
-us,scenario_064,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It subtracted both the repealed $1,796 domestic production deduction and a $3,870 QBI deduction (the correct QBI is $510), deducted the dependent's $18 IRA contribution on the filers' return, and used a $32,500 standard deduction. It also applied a $2,000 CTC instead of $2,200."
-us,scenario_064,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Its $4,464 is $22.54 too high, which is about $188 too little in deductions at 12%. It dropped part of the small deductions (the $108.19 IRA deduction, the $510 QBI deduction, or the cents of the $13,899.29 overtime premium) that bring taxable income to $67,812.12."
-us,scenario_064,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income on top of the $19,350 partnership income, deducted half of an SE tax that doesn't exist, subtracted the repealed $1,796 domestic production deduction, took a $3,310 QBI deduction instead of $510, and omitted the $13,899 overtime deduction."
-us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It deducted the full $18,235 capital loss instead of the $3,000 limit, omitted the overtime deduction, and denied the CTC/ODC on income grounds, although the $400,000 joint phaseout is far above AGI. Its final $14,629.40 also adds an unexplained $7,000 to its own $7,629.40 tax."
-us,scenario_064,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It added the $12,150 'child tax credit' to tax instead of subtracting it, and it deducted both itemized medical expenses and the standard deduction. It also deducted the full capital loss and the dependents' retirement contributions, and it never applied the overtime deduction or the correct $3,200 of CTC/ODC."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"It got the components right: AGI of $114,422, the $13,899 overtime deduction, $510 QBI and $3,200 in credits. Those give about $7,630 − $3,200 ≈ $4,430. It then added about $1,529 of unexplained 'adjustments' to reach $5,959."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It added the $14,000 self-employment partnership income on top of the $19,350 partnership/S-corp income, but the $14,000 is part of that figure, so it was counted twice. It also deducted the repealed $1,796 domestic production deduction and took QBI of $3,310 instead of $510. As a result, taxable income was $77,217 instead of $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It said outright that the $16,800 farm loss does not reduce income, but the farm loss is netted above the line. It also subtracted the repealed $1,796 domestic production deduction. That made AGI $129,408 instead of $114,421."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $13,899 FLSA overtime premium to wages, even though wages already include it, and it never took the overtime deduction. It deducted the full $18,235 capital loss instead of $3,000. It used a $14,600 standard deduction instead of the 2026 MFJ $32,200. It then applied arbitrary unnamed credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It answered zero with no computation. Taxable income is $67,812 after the standard, overtime and QBI deductions, and tax on that is $7,641. The $3,200 of child and other-dependent credits does not fully offset that tax, so liability is not zero."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It charged SE tax on the $14,000 partnership income and deducted half of it, but the $16,800 farm loss makes net self-employment earnings negative. It left out the $13,899 OBBBA overtime deduction. It also capped the nonrefundable CTC at $1,072 even though tax liability was large enough, then added an arbitrary $1,000."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It left out the $13,899 overtime deduction. It added $1,978 of SE tax to income tax, but there is no SE tax here because the farm loss makes net SE earnings negative. It never subtracted the $3,000 of credits it named, and its answer is higher than its own pre-credit tax."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out the $13,899 overtime deduction and deducted the repealed $1,796 DPAD plus a half-SE-tax deduction that doesn't apply. It added an unnamed 'additional tax on other income' and subtracted only $2,500 of credits instead of $3,200. Its $11,200 answer exceeds the tax it computed before credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It never took the $13,899 OBBBA overtime deduction, and it used a $30,000 standard deduction instead of $32,200. It also took a $1,560 self-employed health insurance deduction and a half-SE-tax deduction that don't apply, and it used a $2,000 CTC instead of $2,200."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It left out the $13,899 overtime deduction and deducted the repealed $1,796 DPAD and half of SE tax. It then called the CTC negligible and added unspecified Medicare/NIIT amounts, which pushed the answer to $12,500, above its own $9,300 tax estimate."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Its $128.5k AGI counts the $14,000 self-employment partnership income twice, on top of the $19,350 partnership income. It also never subtracted the $13,899 overtime deduction and took QBI of $3.3k instead of $510, so taxable income was $92.9k instead of $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income twice, deducted the repealed $1,796 DPAD and left out the $13,899 overtime deduction. It also gave the 18-year-old a $2,000 CTC, but that dependent qualifies only for the $500 other-dependent credit."
+us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied pre-TCJA sunset law: a $15,900 standard deduction, $26,000 of personal exemptions, a 15% bracket and a $1,000 CTC. OBBBA made the $32,200 standard deduction, the 12% bracket and the $2,200 CTC permanent. It also left out the overtime deduction and the other-dependent credits, and it subtracted health premiums from wages."
+us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used pre-TCJA rules, with personal exemptions and a $1,000 CTC phasing out at $110,000. Under OBBBA, 2026 has the $32,200 standard deduction, the $13,899 overtime deduction, a $2,200 CTC and $500 other-dependent credits, all with a $400,000 phaseout threshold."
+us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its $114,422 AGI is correct. It then left out the $13,899 overtime deduction, deducted the repealed $1,796 DPAD, used a $30,750 standard deduction instead of $32,200 and used a $2,000 CTC instead of $2,200. Taxable income came out at $81,366 instead of $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used pre-TCJA law, with $25,250 of personal exemptions and a $1,000 CTC. It took the $19,648 employer premium out of wages and itemized medical expenses that included premiums. It also treated the $1,796 DPAD as QBI and never took the $32,200 standard deduction or the $13,899 overtime deduction."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"Its $11,090 answer is higher than the correct $7,641 tax before credits. That fits leaving out the $13,899 overtime deduction and never subtracting the $3,200 of CTC and other-dependent credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It used pre-TCJA rules: $26,000 of personal exemptions, a $16,200 standard deduction and a $1,000 CTC. It added the $2,000 10% early-withdrawal tax, which is not part of income tax before refundable credits. It also took ESI premiums out of wages, deducted the DPAD and left out the overtime deduction."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used personal exemptions and a $1,000 CTC under pre-TCJA law, cut AGI to $94,756 by taking out employer premiums, and never took the $13,899 overtime deduction. It also skipped the $2,200 OBBBA CTC and the two $500 other-dependent credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no derivation. Its $11,940 is higher than the correct $7,641 tax before credits, which fits leaving out the $13,899 overtime deduction and not subtracting the $3,200 of credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its reasoning lists only AGI and the standard deduction. Adding back $3,200 of credits gives about $9,415 of tax, which corresponds to roughly $81,700 of taxable income. That is the reference's $67,812 plus the $13,899 overtime deduction it left out."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $4,324 is $117 below the reference. At the 12% rate, that equals about $980 of extra deductions beyond the $32,200 standard, $13,899 overtime and $510 QBI deductions that produce $67,812.12 of taxable income."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"Its $4,057 is $384 below the reference. At the 12% rate, that equals about $3,200 of extra deductions, which fits taking QBI on the gross partnership income or the repealed DPAD instead of the $510 QBI on the $2,550 net of the farm loss."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,other,False,"It added the $2,000 10% early-distribution tax, which this measure excludes. It deducted half of an SE tax that doesn't arise because net SE earnings are negative, and it used a $2,000 CTC instead of $2,200. Its list of deductions also leaves out the $13,899 overtime deduction."
+us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so no answer was available to score."
+us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so no answer was available to score."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It assumed retirement, medical and credit items wipe out the tax. The medical expenses aren't deductible because the $32,200 standard deduction is larger, and $67,812 of taxable income produces $7,641 of tax, which is more than the $3,200 of credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It assumed deductions and credits remove all tax liability. After the standard, overtime and QBI deductions, $67,812 of taxable income still produces $7,641 of tax, and only $3,200 of nonrefundable credits apply."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Adding back $3,200 of credits, its $2,219 implies about $49,300 of taxable income. That is roughly $18,500 below the correct $67,812.12, an over-deduction about the size of the full $18,235 capital loss compared with the $3,000 annual limit."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It got the structure right but was $44 high, which equals about $367 less in deductions than the reference's combined $32,200 standard, $13,899.29 overtime, $510 QBI and $108.19 IRA deductions."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Its explanation subtracts only the qualifying-child credit. Its $5,455 is about $1,000 above the reference because it left out the two $500 credits for other dependents, for the 27-year-old disabled dependent and the 18-year-old."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It cut AGI to $92,978 by subtracting the $19,648 employer-sponsored insurance premium from wages, even though the reported wages are not reduced by it. It also deducted the repealed $1,796 domestic production deduction. Together these understated AGI by about $21,444."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $5,570 is $1,129 above the reference, about $9,404 of extra taxable income. That matches counting the $14,000 SE partnership income twice, offset by the repealed $1,796 DPAD and a $3,310 QBI deduction instead of $510."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It cut AGI to $112,626 by deducting the repealed $1,796 domestic production deduction. It also claimed $1,500 of other-dependent credits, but only Dependent 1 and Dependent 2 qualify, for $1,000."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no derivation, and its $13,500 is far above the correct $7,641 tax before credits. That fits leaving out the $13,899 overtime deduction and the $3,200 of CTC and other-dependent credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Its $114,422 AGI is correct, but it applied TCJA-sunset law: a $17,000 standard deduction, $27,000 of exemptions, a 15% bracket and a phased $1,000 CTC. Under OBBBA, 2026 has the $32,200 standard deduction, the $13,899 overtime deduction, a $2,200 CTC and two $500 other-dependent credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It added the $14,000 SE partnership income on top of the $19,350 partnership income, counting it twice. It took QBI of $3,310 instead of $510 and used a $30,700 standard deduction instead of $32,200. Taxable income came out at $80,513 instead of $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income twice and took QBI of $3,310 instead of $510. It also added a $2,000 early-distribution penalty, which is not part of income tax before refundable credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted $21,208 of ESI and health premiums from wages, counted the $14,000 SE partnership income twice and deducted half of an SE tax that doesn't apply. It also left out the overtime deduction and claimed only a $2,000 CTC, with no $500 credits for the two older dependents."
+us,scenario_064,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"It added a $2,000 10% early-withdrawal tax on the 401(k) distribution, but that tax is not part of income tax before refundable credits. It also deducted the repealed $1,796 DPAD instead of the $510 QBI deduction. Without the penalty, its own figure would have been about $3,957."
+us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income twice and deducted the dependent's 401(k) and IRA contributions on the parents' return. It took a $6,670 QBI deduction that ignores the farm loss and never took the $13,899 overtime deduction. It also denied the 18-year-old the $500 other-dependent credit and used a $2,000 CTC."
+us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used gross wages of $97,295 without excluding the $2,315 traditional 401(k) deferral. It deducted the repealed $1,796 domestic production deduction instead of the $510 QBI deduction. Together these overstated taxable income by $1,030, giving $68,842 instead of $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It answered zero with no derivation. Taxable income is $67,812 after the standard, overtime and QBI deductions, which produces $7,641 of tax, and $3,200 of nonrefundable credits leave $4,441."
+us,scenario_064,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It deducted the repealed $1,796 DPAD and took about $3,870 of QBI instead of $510, which is 20% of the $2,550 partnership income net of the farm loss. It also used a $2,000 CTC instead of the 2026 $2,200 amount."
+us,scenario_064,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It was $23 high, which equals about $190 of taxable income above the reference's $67,812.12. That comes from slightly under-applying one of the standard, overtime, QBI or IRA deductions, not from missing a major component."
+us,scenario_064,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It counted the $14,000 SE partnership income twice, deducted half of an SE tax that doesn't arise and deducted the repealed $1,796 DPAD plus $3,310 of QBI. It also never took the $13,899 overtime deduction, so taxable income was $89,947 instead of $67,812."
+us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It deducted the full $18,235 capital loss instead of the $3,000 limit and left out the overtime deduction. It claimed the household's income exceeds the credit thresholds, but the CTC phaseout starts at $400,000 for joint filers. It then added $7,000 with no explanation to its $7,629 tax."
+us,scenario_064,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It deducted the full capital loss, took both itemized medical deductions and the standard deduction, and invented a $12,150 child tax credit. It then added that credit to tax instead of subtracting it, when the correct CTC plus other-dependent credits total $3,200."
us,scenario_064,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It equated the refundable credit with the full $2,000 CTC for Child 1, skipping the ACTC computation entirely — the CTC first offsets federal income tax liability and only the unused remainder becomes refundable, capped at $1,700 per child and at 15% of earned income over $2,500. With $97,295 of wages plus $20,000 of taxable 401(k) distributions and $19,350 of partnership income, MFJ tax before refundable credits far exceeds $2,000, so the entire CTC is consumed nonrefundably and nothing spills into the refundable column. Its own AGI estimate of ~$120,610 also double-counted, adding partnership income while ignoring that the capital loss deduction is capped at $3,000."
us,scenario_064,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its reasoning reached the correct conclusion — 'with significant federal tax owed, CTC is nonrefundable' — and it then submitted the $1,700 statutory ACTC ceiling anyway, converting a per-child maximum into an entitlement. The ACTC equals only the CTC remaining after the credit is applied against federal income tax liability; here liability on ~$98,810 of income absorbs the full $2,000 for the one under-17 child, so the refundable amount is $0."
us,scenario_064,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It counted Dependent 2 (age 18) as a CTC qualifying child, but the credit requires age under 17 at year end, so that dependent yields only the $500 nonrefundable other-dependent credit rather than a $1,700 refundable share. It then paid out the per-child ACTC ceiling without applying the rule that only the CTC unused against tax liability is refundable; MFJ tax before refundable credits on ~$98,810 of income exceeds $2,000, so even the single qualifying child's credit is fully nonrefundable and the correct total is $0. Checking the $400,000 phaseout threshold was irrelevant — the binding constraint is the liability offset, not the income phaseout."
@@ -5738,24 +5803,25 @@ us,scenario_064,head_medicare_eligible,glm-5.3,parse_contract_failure,missing_ou
us,scenario_064,head_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_064,local_income_tax,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_064,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_064,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It computed the FICA pieces correctly ($6,032.29 + $1,410.78), then added a made-up 3% Wisconsin employee payroll tax ($2,918.85). Wisconsin has no mandatory employee state payroll tax. It then inflated the total again, without explanation, to $12,318.64. The correct answer is FICA alone: $7,443.07."
-us,scenario_064,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"It declined to calculate employee Social Security and Medicare tax and submitted 0, even though the head has $97,295 of wages. Those wages produce $6,032.29 of Social Security tax plus $1,410.78 of Medicare tax, for $7,443.07."
-us,scenario_064,payroll_tax,claude-sonnet-5,llm_error,other,False,"Its reasoning reached the correct total: 6.2% and 1.45% on $97,295, giving $7,443.07. It then submitted the unrelated figure $7,407.36 and called it rounding. This was a transcription error in the final step, not an error in the payroll-tax rules."
-us,scenario_064,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the head's $1,560 health insurance premiums from wages and used $95,735 as the FICA base. Nothing in the facts makes those premiums a pre-tax payroll deduction, so the full $97,295 is subject to FICA, giving $7,443.07."
-us,scenario_064,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It cut the FICA base to $75,332 by subtracting both the $19,648 ESI premium and the $2,315 traditional 401(k) deferral. Traditional 401(k) elective deferrals stay in FICA wages under IRC 3121(v)(1)(A), and the ESI premium is not a listed salary reduction. FICA applies to all $97,295 of wages."
-us,scenario_064,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It assumed the $19,648 employer-sponsored insurance premium was a pre-tax cafeteria-plan deduction and taxed only $77,647. The facts list gross wages of $97,295 with no salary reduction, so 7.65% applies to $97,295, giving $7,443.07."
-us,scenario_064,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It treated the $19,648 ESI premium as a pre-tax deduction from FICA wages, which left a base of $77,647 and a total of $5,940. The employee FICA base is the full $97,295 of listed gross wages, which gives $6,032.29 + $1,410.78 = $7,443.07."
-us,scenario_064,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It subtracted pre-tax health insurance (the $19,648 ESI premium) from wages before applying FICA, which produced $5,940. No salary reduction is listed, so Social Security and Medicare apply to the full $97,295 of wages, giving $7,443.07."
-us,scenario_064,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"It reduced FICA wages to $77,647 by treating the $19,648 ESI premium as a pre-tax health deduction. The correct base is the full $97,295 of gross wages: 7.65% of that is $7,443.07."
-us,scenario_064,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there was nothing to score. The correct value is 7.65% of the head's $97,295 wages, or $7,443.07."
-us,scenario_064,payroll_tax,glm-5.3,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there was nothing to score. The correct value is $6,032.29 of Social Security tax plus $1,410.78 of Medicare tax, or $7,443.07."
-us,scenario_064,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"At 7.65%, its $12,306 implies a FICA base of about $160,900, far above the head's $97,295 of wages. It folded non-wage income into the base: partnership/S-corp income, self-employment income, and 401(k) distributions are not employee FICA wages. Only the $97,295 is taxable, giving $7,443.07."
-us,scenario_064,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"At 7.65%, its $8,659 implies a wage base of about $113,200 instead of the $97,295 of listed gross wages. It added income on top of wages, such as the $13,899 FLSA overtime premium, which the listed gross wages already include. Employee FICA on $97,295 is $7,443.07."
-us,scenario_064,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It applied 7.65% to $77,647 after treating the $19,648 employer-sponsored insurance premium as a pretax salary reduction. No such reduction is listed, so the FICA base is the full $97,295, giving $7,443.07."
-us,scenario_064,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for payroll_tax, so there was nothing to score. The correct value is 7.65% of the head's $97,295 wages, or $7,443.07."
-us,scenario_064,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"It submitted 0 with no reasoning, which ignores the head's $97,295 of wages. Those wages carry $6,032.29 of employee Social Security tax and $1,410.78 of employee Medicare tax, for $7,443.07."
-us,scenario_064,payroll_tax,ox-alpha,llm_error,payroll_tax_base,False,"It subtracted the $2,315 traditional 401(k) deferral and taxed only $94,980. Under IRC 3121(v)(1)(A), elective 401(k) deferrals remain subject to Social Security and Medicare tax, so the base is $97,295 and the total is $7,443.07."
-us,scenario_064,payroll_tax,qwen3.8-max,llm_error,other,False,"Its reasoning computed the correct pieces, $6,032.29 of Social Security tax and $1,410.78 of Medicare tax, which add to $7,443.07. It then submitted the unrelated figure $8,529.65. The error was in adding up and reporting the final number, not in the payroll-tax rules."
+us,scenario_064,payroll_tax,claude-fable-5.2,llm_error,payroll_tax_base,False,"It subtracted the $2,315 traditional 401(k) deferral from the FICA base, using $94,980. Elective 401(k) deferrals are FICA wages under IRC §3121(v)(1)(A), so the base is the full $97,295 and the tax is $7,443.07."
+us,scenario_064,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It computed the FICA amounts correctly ($6,032.29 + $1,410.78) but then added an invented 3% Wisconsin employee payroll tax. Wisconsin has no mandatory employee state payroll tax. It then replaced even that subtotal with an unexplained $12,318.64 instead of $7,443.07."
+us,scenario_064,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"It declined to compute employee FICA and returned 0. The head's $97,295 of wages are fully subject to 6.2% Social Security and 1.45% Medicare, which totals $7,443.07."
+us,scenario_064,payroll_tax,claude-sonnet-5,llm_error,other,False,"Its own reasoning reached the correct total of $6,032.29 + $1,410.78 = $7,443.07. It then submitted a mismatched $7,407.36, which is a transcription or arithmetic error in the final step."
+us,scenario_064,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It subtracted the $1,560 'health insurance premiums excluding Medicare Part B' from wages to get a $95,735 FICA base. That figure is a medical expense input, not a pre-tax salary reduction, so the base stays at the full $97,295 of gross wages."
+us,scenario_064,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"It reduced the FICA base to $75,332 by subtracting both the $19,648 employer-sponsored insurance premium and the $2,315 traditional 401(k) deferral. 401(k) deferrals are FICA wages, and the prompt never says the premiums were withheld from salary pre-tax. The base is the full $97,295."
+us,scenario_064,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It treated the $19,648 employer-sponsored insurance premium as a pre-tax salary reduction and taxed only $77,647. The prompt gives no cafeteria-plan withholding, so FICA applies to the full $97,295, giving $7,443.07."
+us,scenario_064,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It subtracted the $19,648 employer-sponsored insurance premium from gross wages to get a FICA base of $77,647. The prompt never says this premium was a pre-tax payroll deduction, so 7.65% applies to the full $97,295."
+us,scenario_064,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"It reduced taxable wages by pre-tax health insurance deductions, which yields $5,940 = 7.65% × $77,647, i.e. $97,295 minus the $19,648 premium. FICA applies to the full $97,295 of listed gross wages."
+us,scenario_064,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"It subtracted the $19,648 employer-sponsored insurance premium as a pre-tax health deduction, which gives a $77,647 base. The correct base is the full $97,295 of gross wages, so the tax is $7,443.07."
+us,scenario_064,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"It gave no payroll_tax value or explanation, so there was no answer to score against the $7,443.07 reference."
+us,scenario_064,payroll_tax,glm-5.3,parse_contract_failure,missing_output,False,"It gave no payroll_tax value or explanation, so there was no answer to score against the $7,443.07 reference."
+us,scenario_064,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"Its $12,306 implies a 7.65% base of about $160,900, which is roughly $63,600 more than the $97,295 of listed wages. It therefore added non-wage items, such as partnership or self-employment income, 401(k) distributions or the overtime premium, to the employee FICA base. Only W-2 wages count toward employee FICA."
+us,scenario_064,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"Its $8,659 implies a 7.65% base of about $113,200, which is roughly $15,900 more than the head's $97,295 of wages. It therefore inflated the base with an extra amount, such as the overtime premium already included in gross wages or non-wage income. The correct FICA on $97,295 is $7,443.07."
+us,scenario_064,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It applied 7.65% to $77,647 after treating the $19,648 employer-sponsored insurance premium as a pre-tax salary reduction. The prompt lists no such withholding, so the base is the full $97,295 and the tax is $7,443.07."
+us,scenario_064,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no payroll_tax value or explanation, so there was no answer to score against the $7,443.07 reference."
+us,scenario_064,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"It returned 0 with no reasoning, which ignores the head's $97,295 of wages. Those wages are subject to 6.2% Social Security and 1.45% Medicare, totaling $7,443.07."
+us,scenario_064,payroll_tax,ox-alpha,llm_error,payroll_tax_base,False,"It subtracted the $2,315 traditional 401(k) deferral and taxed $94,980. Elective 401(k) deferrals are FICA wages under IRC §3121(v)(1)(A), so the full $97,295 is taxed and the total is $7,443.07."
+us,scenario_064,payroll_tax,qwen3.8-max,llm_error,other,False,"Its reasoning correctly computed $6,032.29 Social Security and $1,410.78 Medicare on $97,295, with no Additional Medicare Tax, which sums to $7,443.07. It then submitted an inconsistent $8,529.65 that matches none of its stated components."
us,scenario_064,reduced_price_school_meals_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_064,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"It built the base by adding the full $19,350 partnership/S-corp share to the $14,000 self-employment partnership earnings, double-counting the same K-1 income and treating non-SE distributive share as SE earnings, so its $16,550 base stayed positive instead of the non-positive net earnings the -$16,800 farm loss produces. It then halved its own $2,337.65 result on the theory that only 50% is owed, confusing the one-half-of-SE-tax above-the-line income tax deduction under section 164(f) with a reduction of the SE tax itself. The submitted $1,965.48 matches neither of its two computations and is an unexplained fudge."
us,scenario_064,self_employment_tax,claude-opus-5,llm_error,payroll_tax_base,False,"It correctly identified that the -$16,800 farm loss offsets the $14,000 partnership self-employment earnings and zeroes the base, then overrode that with an invented claim that PolicyEngine taxes the $14,000 in isolation. Section 1402(a) net earnings aggregate all of a person's trades and businesses, so the Schedule F farm loss is netted against the partnership SE earnings and leaves nothing to tax."
@@ -5775,76 +5841,78 @@ us,scenario_064,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_ou
us,scenario_064,spouse_medicaid_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_064,spouse_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_064,spouse_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_064,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Read Dependent 1's generic 'is disabled' flag as meeting SSI's disability criterion and paid the full $994/month FBR. The engine has is_ssi_aged_blind_disabled=False for Dependent 1 and every other member, so no one is categorically eligible and SSI is $0."
-us,scenario_064,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Treated the 'is disabled' flag as SSI-qualifying disability for Dependent 1 and awarded $994 × 12. The engine's is_ssi_aged_blind_disabled is False for Dependent 1, so the deeming analysis never comes into play and SSI is $0."
-us,scenario_064,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Took Dependent 1's general disability flag as satisfying SSI's disability requirement and paid an estimated $991/month FBR. In the engine, Dependent 1 is not SSI aged/blind/disabled, so the benefit is $0 whatever the income or resources."
-us,scenario_064,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-disabled and paid a one-third-reduced FBR ($7,928). Dependent 1's is_ssi_aged_blind_disabled is False, so there is no SSI at all. The model also assumed in-kind support and maintenance (ISM) that the facts do not list."
-us,scenario_064,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted Dependent 1 as SSI-disabled and applied the one-third reduction for living in another person's household to get $7,920. The engine finds no member meeting SSI's aged/blind/disabled test, so SSI is $0 before any federal benefit rate or reduction applies."
-us,scenario_064,ssi,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Treated Dependent 1's 'is disabled' flag as SSI-eligible disability and subtracted an ISM reduction from a $991 FBR to get $7,688.04. Dependent 1 has is_ssi_aged_blind_disabled=False in the engine, so the engine's SSI is $0."
-us,scenario_064,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Called Dependent 1's disability 'qualifying' for SSI and paid the projected maximum federal benefit ($11,880). The engine does not treat Dependent 1 as SSI-disabled, so no member is categorically eligible and SSI is $0."
-us,scenario_064,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on the strength of the general disability flag and added a Wisconsin state supplement to reach $11,836. Dependent 1's is_ssi_aged_blind_disabled is False, so neither federal SSI nor any supplement is paid, and SSI is $0."
-us,scenario_064,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"Correctly found that no one else is 65+, blind, or disabled, but read Dependent 1's 'is disabled' flag as SSI-qualifying and paid an estimated $991/month FBR. The engine's SSI status for Dependent 1 is not aged/blind/disabled, so SSI is $0."
-us,scenario_064,ssi,glm-5.3,parse_contract_failure,missing_output,False,"Did not return an SSI value or any explanation, so nothing was available to score. The engine's value is $0 because no household member is SSI aged, blind, or disabled in the engine."
-us,scenario_064,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"Treated the disabled adult dependent as meeting SSI's disability criterion and paid the 2026 individual rate of $994 × 12. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, so SSI is $0."
-us,scenario_064,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Equated the 'is disabled' flag with SSI disability eligibility and paid the $994/month maximum. In the engine, Dependent 1 does not meet SSI's aged/blind/disabled requirement, so the benefit is $0."
-us,scenario_064,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"Correctly ruled out parental deeming for a 27-year-old, but assumed Dependent 1 is SSI-disabled because the person is flagged disabled. The engine's is_ssi_aged_blind_disabled is False for Dependent 1, so SSI is $0 rather than $11,928."
-us,scenario_064,ssi,gpt-6-luna,prompt_ambiguity,age_disability,False,"Took the 'marked disabled' attribute as satisfying SSI's disability test and paid the full 2026 FBR. Dependent 1 is not SSI aged/blind/disabled in the engine, so no SSI is payable and the engine's value is $0."
-us,scenario_064,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"Assumed the disabled 27-year-old qualifies for SSI and paid $994 × 12. The engine finds Dependent 1 does not meet SSI's aged/blind/disabled criterion, so SSI is $0."
-us,scenario_064,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Focused on deeming and countable assets but never questioned whether Dependent 1 meets SSI's disability status. That status is False in the engine (is_ssi_aged_blind_disabled=False), which makes SSI $0."
-us,scenario_064,ssi,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"Paid the full 2026 federal SSI benefit on the assumption that the general disability flag makes Dependent 1 categorically eligible. The engine sets is_ssi_aged_blind_disabled=False for every member, so SSI is $0."
-us,scenario_064,ssi,inkling,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-disabled, applied a one-third reduction for living in another person's household, and added a WI supplement to get $7,952. Dependent 1 is not SSI aged/blind/disabled in the engine, so SSI is $0. The model also applied a reduction for in-kind support that the facts do not list."
-us,scenario_064,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"Did not return an SSI value or any explanation, so nothing was available to score. The engine's value is $0 because no member meets SSI's aged/blind/disabled criterion in the engine."
-us,scenario_064,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Correctly excluded Head and Spouse, but called Dependent 1 'categorically SSI-eligible' because of the disability flag and paid $994 × 12. The engine's is_ssi_aged_blind_disabled is False for Dependent 1 as well, so SSI is $0."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Invented a ~$989 half-SE-tax deduction even though net SE earnings are negative after the farm loss. Added back $2,500 under a $500 capital-loss cap that the engine does not apply, and taxed income on the old brackets with 4.4% ending at $38,190 ($4,964). It then cut the result to $2,699 with ~$2,265 of nonrefundable credits (itemized deduction credit and others) that do not apply; this household has zero Wisconsin nonrefundable credits."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Overstated federal AGI at $126,626 by adding the $14,000 SE partnership income on top of the $19,350 partnership/S-corp income it is part of, and by subtracting the $1,796 domestic production deduction. It then added a $2,500 capital-loss addback. That gave ~$117,700 of taxable income instead of $101,968.14, taxed on the old, narrower 4.4% bracket."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Started from a ~$104,434 'federal taxable income' that subtracts the full $18,235 capital loss, instead of federal AGI of $114,421.41 with the loss capped at $3,000. It cited joint rates of 5.3%-7.65%, which are only Wisconsin's top two rates; the 2026 joint schedule starts at 3.5% and 4.4%, and this household's income reaches only the 5.3% bracket. It never applied the $8,953.27 sliding-scale deduction and five $700 exemptions, so its $4,512 is not derived from the 3.5%/4.4%/5.3% schedule."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Skipped the Wisconsin calculation and defaulted to $0. Wisconsin taxable income is $101,968.14 after the $8,953.27 sliding-scale deduction and $3,500 of exemptions, which produces $4,599.62 of tax."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Left the $2,315 401(k) deferral and the $108 IRA deduction in AGI ($116,845), counted four $700 exemptions instead of five, and used the old bracket with 4.4% ending at $36,840. It then subtracted ~$875 for a school property tax credit and itemized deduction credit. The household has no property tax, no rent and no qualifying itemized deductions, so nonrefundable credits are zero."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Taxed ~$100,000 on the old joint brackets (4.4% ending at $38,190) to get $4,784, then raised the answer to $5,074 with no supporting arithmetic while describing exemptions as credits. Under the 2026 schedule's widened 4.4% bracket, the correct $101,968.14 of taxable income produces $4,599.62."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Added back $2,500 of capital loss under a $500 Wisconsin cap, whereas the engine carries the full $3,000 federally limited loss into WI AGI ($114,421.41). That raised WI AGI to $116,922, cut the sliding-scale deduction to ~$8,000 instead of $8,953.27, and lifted taxable income to $105,422 versus $101,968.14."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Dropped the head's $108 IRA deduction as phased out, but MAGI of ~$114k is below the 2026 joint active-participant phase-out, so it is fully deductible. It also invented a ~$990 SE-tax deduction, added back $2,500 of capital loss, and phased the standard deduction down linearly to $3,957 instead of the $8,953.27 sliding-scale amount. It then taxed $108,583 on the old brackets and subtracted a made-up $520 credit."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Overstated AGI at $126,626 by counting the $14,000 SE partnership income on top of the $19,350 partnership income and by subtracting the $1,796 domestic production deduction, which is not an AGI adjustment. It used a ~$14,000 standard deduction instead of $8,953.27 and four exemptions instead of five. Its $4,200 does not follow from its own $109,826 taxable income under the 3.5%/4.4%/5.3% schedule."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Put Wisconsin taxable income at ~$125,000, about $23,000 above the correct $101,968.14. It counted only three dependent exemptions instead of five $700 exemptions (head, spouse and three dependents), then applied the old, narrower 4.4% bracket to get $6,200."
-us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Got roughly the right taxable income (~$100,000) but replaced Wisconsin's joint schedule with a made-up 4% rate on the first ~$23,000 and 5.3% on the rest. The 2026 schedule applies 3.5%, then a 4.4% bracket widened by 2025 law that runs to roughly $69,000 for joint filers, then 5.3%. On $101,968.14 that produces $4,599.62."
-us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Subtracted the $1,796 domestic production deduction as a Wisconsin subtraction, which Wisconsin does not have. It also took a flat $25,000 standard deduction with no sliding-scale phase-down; at this income the deduction is only $8,953.27. Taxable income fell to $82,566 instead of $101,968.14."
-us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Its $119,774.69 taxable income is about $17,800 too high; WI AGI equals federal AGI of $114,421.41, less $8,953.27 and $3,500, giving $101,968.14. Its $5,829.93 matches the old joint brackets with 4.4% ending near $38,190, not the 2026 schedule's widened 4.4% bracket."
-us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Treated the $3,500 of personal exemptions as a dollar-for-dollar tax credit instead of a deduction from income. It also used the old 2022 rates of 3.54% and 4.65% and a $13,197 standard deduction instead of the $8,953.27 sliding-scale amount. Taken as deductions, the $700 exemptions bring taxable income to $101,968.14, taxed at $4,599.62."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Took a $13,506 sliding-scale deduction instead of $8,953.27, under-applying the phase-down at ~$114k of WI AGI, which gave $97,576 of taxable income instead of $101,968.14. It then subtracted $150 of dependent credits, although the household has no Wisconsin nonrefundable credits."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Gave no computation. Its $4,652.12 is $52.50 above the correct tax, which equals about $990 of extra taxable income over the $101,968.14 left after federal AGI of $114,421.41, the $8,953.27 sliding-scale deduction and five $700 exemptions."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Started from $92,978 of AGI, $21,443 short of $114,421.41. The correct AGI includes wages net of the $2,315 deferral, the $20,000 taxable 401(k) distribution and $19,350 of partnership income, less the $16,800 farm loss, a $3,000 capital loss and the $108 IRA deduction. Its $78,969 taxable income therefore understates the correct $101,968.14."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Reported $82,256 of taxable income, about $19,700 below $101,968.14, because it over-applied the standard deduction instead of the phased-down $8,953.27. Its $4,356 also does not follow from that base under the 3.5%/4.4%/5.3% schedule."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Gave no derivation. $6,950 is $2,350 above the correct tax, equal to taxing about $44,000 more than the $101,968.14 left after the $8,953.27 sliding-scale deduction and $3,500 of exemptions, so it skipped deductions or counted income twice."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Gave no computation. $4,125 is $475 below the correct tax, equal to about $9,000 of extra deductions or invented credits. The household's only offsets are the $8,953.27 sliding-scale deduction and $3,500 of exemptions; it has no nonrefundable credits."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Gave no computation. $3,212 is $1,388 below the correct tax, which fits an unreduced standard deduction or invented nonrefundable credits. At WI AGI of $114,421.41 the sliding-scale deduction is only $8,953.27, and no nonrefundable credits apply."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Gave no computation. $2,490 is $2,110 below the correct tax, which reflects deductions or nonrefundable credits the household does not have. Taxable income is $101,968.14 after the $8,953.27 phased-down deduction and $3,500 of exemptions, with zero credits."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"Included a $666.67 Wisconsin early-withdrawal penalty, which is not part of income tax before refundable credits, and added back capital loss under a $500 cap that the engine does not apply. It also took a medical-insurance premium subtraction that the engine does not calculate from these inputs. It claimed a married couple credit, which is $0 because the spouse has no earned income."
-us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Returned no value for state_income_tax_before_refundable_credits. The correct figure is $4,599.62 on $101,968.14 of Wisconsin taxable income."
-us,scenario_064,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"Returned no value for state_income_tax_before_refundable_credits. The correct figure is $4,599.62 on $101,968.14 of Wisconsin taxable income."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Claimed that deductions and retirement adjustments wipe out the tax. At $114,421.41 of WI AGI the sliding-scale deduction is phased down to $8,953.27 and exemptions are only $3,500, leaving $101,968.14 taxable and $4,599.62 of tax with no nonrefundable credits."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed deductions and credits exceed the tax. The joint standard deduction is largely phased out ($8,953.27) at $114,421.41 of WI AGI, exemptions total $3,500, and no nonrefundable credits apply, so $4,599.62 of tax remains."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Used a $15,166 standard deduction instead of $8,953.27 and only two $700 exemptions instead of five. It then subtracted a $472 itemized-deduction credit, but there is no mortgage interest input and medical costs of $8,340 fall below the 7.5%-of-AGI floor (~$8,582), so no credit arises."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Gave no computation. $2,800 is $1,800 below the correct tax, which reflects invented nonrefundable credits or overstated deductions. Wisconsin taxable income is $101,968.14 after the $8,953.27 sliding-scale deduction and $3,500 of exemptions, with no credits."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $4,872 matches roughly the correct taxable income taxed on the old joint brackets, where 4.4% ends near $38,000-$40,000. The 2026 schedule widens the 4.4% bracket to roughly $69,000, which brings tax on $101,968.14 down to $4,599.62."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Treated exemptions as credits and subtracted other offsets, landing $529 below the correct tax. Wisconsin's $700 exemptions reduce income and are already counted in the $101,968.14 of taxable income, and the household has no nonrefundable credits."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Took a $2,340 health-insurance subtraction that the engine does not calculate from these inputs, and added back $2,500 of capital loss that the engine does not add back. Those two roughly cancel, so its $3,254.36 implies about $25,000 less taxable income than $101,968.14, which fits failing to phase the joint standard deduction down to $8,953.27."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Gave no computation. $5,654 is $1,054 above the correct tax, which overstates taxable income by about $20,000 compared with the $101,968.14 left after the $8,953.27 sliding-scale deduction and five $700 exemptions."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Started from the right ~$114,422 AGI, but its $4,200 is $400 too low, which implies about $7,500 of deductions beyond the $8,953.27 sliding-scale deduction and five $700 exemptions. The joint standard deduction is largely phased out at this income."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Subtracted a ~$4,136 health-premium deduction that the engine does not calculate from these inputs. That lowered income to $110,286 and pushed the sliding-scale deduction up to $10,207.94 instead of $8,953.27, understating taxable income compared with $101,968.14."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,"Gave no computation. $4,800 is $200 high, which fits roughly correct taxable income taxed on the old joint brackets instead of the 2026 schedule with the widened 4.4% bracket, which gives $4,599.62 on $101,968.14."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Gave no computation. $4,500 is $100 below the correct tax, equal to about $1,900 of extra deductions beyond the $8,953.27 sliding-scale deduction and $3,500 of exemptions on $114,421.41 of WI AGI."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Started from the right $114,422 AGI but put the sliding-scale deduction at ~$6,500 instead of $8,953.27 and left out the five $700 exemptions entirely. That produced $107,900 of taxable income instead of $101,968.14."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Used cumulative joint bracket thresholds of $29,350 and $58,700, compared with the 2026 thresholds of $20,150 and $69,260; on its $101,947 taxable-income estimate, those bracket errors raise tax by $12.24. It also started from $114,523 of AGI with a $9,076 deduction instead of $114,421.41 and $8,953.27. The net result is $10.38 too much tax."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Got taxable income nearly right ($102,274 vs $101,968.14), but its $4,876 reflects the old joint brackets with 4.4% ending near $38,000-$40,000. The 2026 schedule's widened 4.4% bracket gives about $4,616 on its own base and $4,599.62 on the correct base."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Understated AGI at $106,225 instead of $114,421.41, skipped the five $700 exemptions, and used a $13,500 standard deduction instead of $8,953.27. It also applied 4.75% and 5.75% rates that it labeled 2024 law but that do not exist; Wisconsin's 2026 joint rates are 3.5%, 4.4%, 5.3% and 7.65%, and this household's income reaches only the 5.3% bracket."
-us,scenario_064,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"Took the federal overtime deduction out of AGI (~$97,723) and then added it back. That deduction comes after AGI, so federal and Wisconsin AGI are both $114,421.41. Its ~$90,000 taxable income is about $12,000 short of $101,968.14."
-us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value for state_income_tax_before_refundable_credits. The correct figure is $4,599.62 on $101,968.14 of Wisconsin taxable income."
-us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Taxed the third bracket at the old 6.27% rate instead of 5.3% and put the sliding-scale deduction at $5,481 instead of $8,953.27. It also started from ~$116,737 of AGI instead of $114,421.41, which inflated the tax to $6,091."
-us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Returned $0 with no reasoning and ignored Wisconsin tax on $101,968.14 of taxable income, which produces $4,599.62."
-us,scenario_064,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"Used a ~$19,800 standard deduction with almost no phase-down (correct: $8,953.27), skipped the $3,500 of exemptions, and taxed the resulting ~$94,600 on the old brackets. The errors partly cancelled and gave $4,481."
-us,scenario_064,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Gave no computation. $4,448 is $152 below the correct tax, equal to about $2,860 of extra deductions beyond the $8,953.27 sliding-scale deduction and $3,500 of exemptions on $114,421.41 of WI AGI."
-us,scenario_064,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Overstated AGI at $127,253 instead of $114,421.41 and used a ~$31,300 federal-style joint standard deduction in place of Wisconsin's $8,953.27 sliding-scale amount. It also left out the five $700 exemptions and used made-up bracket thresholds of $29,430 and $58,860."
-us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Excluded overtime pay from AGI, but the federal overtime deduction comes after AGI and never reduces federal or Wisconsin AGI, so it got $100,523 instead of $114,421.41. It skipped the $3,500 of exemptions, used a $13,500 deduction and an old 4.65% rate, and reported $4,221.67, which does not match its own $3,767 computation."
-us,scenario_064,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Subtracted $11,827 of retirement contributions from AGI that already reflects them: the 401(k) deferral is already out of wages, the IRA is already deducted, and Roth contributions are not deductible. It also deducted medical expenses, which Wisconsin does not allow as a deduction, pushing taxable income well below $101,968.14."
+us,scenario_064,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Treated Dependent 1's generic 'is disabled' flag as meeting SSI's disability criterion, then paid the full $994/month FBR. In the engine, is_ssi_aged_blind_disabled=False for Dependent 1, so no household member is categorically eligible and SSI is $0."
+us,scenario_064,ssi,claude-fable-5.2,prompt_ambiguity,age_disability,False,"Assumed that the 27-year-old marked 'is disabled' is SSI-disabled and paid the full 2026 FBR of $994 × 12. The engine's SSI aged/blind/disabled test is False for Dependent 1, which makes every member categorically ineligible and SSI $0."
+us,scenario_064,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Equated the generic disability flag with SSI disability and awarded $994 × 12. The engine evaluates is_ssi_aged_blind_disabled=False for Dependent 1, so the categorical requirement fails before any income or deeming step applies, and SSI is $0."
+us,scenario_064,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-disabled and paid an estimated $991/month FBR. Dependent 1 does not meet the engine's SSI aged/blind/disabled criterion (is_ssi_aged_blind_disabled=False), so no member is eligible and SSI is $0."
+us,scenario_064,ssi,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Assumed Dependent 1 is SSI-eligible because of the 'is disabled' flag, then applied a one-third in-kind support reduction that the prompt gives no basis for. Dependent 1 fails the engine's SSI aged/blind/disabled test, so SSI is $0 and there is no benefit to reduce."
+us,scenario_064,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as categorically SSI-disabled and paid the FBR minus a one-third living-in-another's-household reduction, giving $660/month. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, so no one qualifies and SSI is $0."
+us,scenario_064,ssi,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Assumed Dependent 1 meets SSI disability and subtracted an ISM reduction of $350.33/month from a $991 FBR. Dependent 1 does not pass the engine's SSI aged/blind/disabled test, so SSI is $0 before any ISM calculation."
+us,scenario_064,ssi,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Read the generic 'is disabled' flag as an SSI-qualifying disability and paid a projected maximum benefit of $11,880. The engine's is_ssi_aged_blind_disabled is False for Dependent 1 and for every other member, so SSI is $0."
+us,scenario_064,ssi,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-eligible on the strength of the generic disability flag and added an assumed Wisconsin state supplement. Dependent 1 fails the engine's SSI aged/blind/disabled criterion, so both the federal benefit and any supplement are $0."
+us,scenario_064,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"Identified Dependent 1 as a disabled adult SSI applicant and paid a COLA-projected $991/month. The engine's SSI aged/blind/disabled flag is False for Dependent 1, so no member is categorically eligible and SSI is $0."
+us,scenario_064,ssi,glm-5.3,parse_contract_failure,missing_output,False,"Returned no value and no explanation for ssi. The correct output was $0, because no household member meets SSI's aged, blind, or disabled criterion in the engine."
+us,scenario_064,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"Assumed the disabled adult dependent qualifies for SSI and paid the $994/month individual rate for 12 months. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, so the categorical test fails and SSI is $0."
+us,scenario_064,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Treated the 27-year-old's generic disability flag as SSI disability and paid the $994/month maximum. Dependent 1 does not meet the engine's SSI aged/blind/disabled criterion, so SSI is $0."
+us,scenario_064,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"Correctly ruled out parental deeming but assumed Dependent 1 is categorically SSI-disabled, so it paid $994 × 12. The engine's is_ssi_aged_blind_disabled is False for Dependent 1, so SSI is $0 at the categorical step."
+us,scenario_064,ssi,gpt-6-luna,prompt_ambiguity,age_disability,False,"Equated 'marked disabled' with SSI eligibility and paid the 2026 federal maximum. Dependent 1 fails the engine's SSI aged/blind/disabled test, and no other member is 65+ or blind, so SSI is $0."
+us,scenario_064,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"Assumed the disabled 27-year-old qualifies for the full $994/month individual rate. The engine treats Dependent 1 as not SSI aged/blind/disabled, so no one is categorically eligible and SSI is $0."
+us,scenario_064,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Correctly ruled out parental resource deeming but treated the generic disability flag as satisfying SSI disability, so it paid $994 × 12. In the engine, is_ssi_aged_blind_disabled=False for Dependent 1, so SSI is $0."
+us,scenario_064,ssi,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"Assumed the disabled adult dependent receives the full 2026 federal SSI benefit. Dependent 1 fails the engine's SSI aged/blind/disabled criterion, which leaves every member categorically ineligible and SSI at $0."
+us,scenario_064,ssi,inkling,prompt_ambiguity,age_disability,False,"Treated Dependent 1 as SSI-disabled, applied a one-third living-in-another's-household reduction, and added a Wisconsin supplement. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, so there is no federal benefit or state supplement, and SSI is $0."
+us,scenario_064,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value and no explanation for ssi. The correct output was $0, because no household member meets SSI's aged, blind, or disabled criterion in the engine."
+us,scenario_064,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"Declared Dependent 1 the only categorically SSI-eligible member and paid $994 × 12. The engine evaluates is_ssi_aged_blind_disabled=False for Dependent 1 as well as for Head and Spouse, so no one is categorically eligible and SSI is $0."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It computed pre-credit tax of $4,964 using the old 4.4% bracket ending at $38,190 instead of the Act 15 band to about $69.3k. It then cut about $2,265 more for an itemized-deduction credit and unexplained 'precision' adjustments, but no Wisconsin nonrefundable credit applies. It also subtracted a $989 SE-tax deduction even though the $16,800 farm loss leaves net SE earnings negative."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Its $126,626 federal AGI adds the $14,000 SE partnership income on top of the $19,350 partnership/S-corp income (it is part of that amount) and subtracts the $1,796 domestic production deduction. It also added back $2,500 of capital loss. Together these inflate taxable income to about $117,700 instead of $101,968.14, which it then taxed at about $5,400."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,state_local_rule,False,"Its $129,408 federal AGI counts the $14,000 SE partnership income as extra income on top of the $19,350 partnership amount. It added a $2,500 capital-loss add-back and used a $4,600 standard deduction instead of $8,953.27, giving $123,800 taxable instead of $101,968.14. It then applied 2025 brackets that predate the Act 15 widening of the 4.4% band."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It cited Wisconsin rates of 5.3%–7.65%, subtracted the full $18,235 capital loss instead of the $3,000 allowed, and never built the 3.5%/4.4%/5.3% joint schedule on $101,968.14 of taxable income. Its $4,512 is an unsupported figure, not the $4,599.62 that schedule produces."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It skipped the Wisconsin calculation and submitted 0. Wisconsin AGI of $114,421.41, less the $8,953.27 sliding-scale standard deduction and $3,500 of exemptions, leaves $101,968.14 taxable and $4,599.62 of tax."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It left out the $2,315 401(k) deferral and $108 IRA deduction from AGI, counted only 4 exemptions instead of 5, and used the old 4.4% bracket ending at $36,840, giving $5,026. It then subtracted about $875 for a school property tax credit and an itemized deduction credit, but no property tax or rent is listed and no Wisconsin nonrefundable credit applies."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Its own math taxed about $100,000 using the old 4.4% bracket ending at $38,190 ($4,784) instead of the Act 15 band to about $69.3k. It then raised the result to $5,074 with no supporting rule. The correct tax on $101,968.14 under the widened bracket is $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It applied a $500 Wisconsin capital-loss cap and added back $2,500, raising Wisconsin AGI to $116,922 instead of the $114,421.41 federal AGI that the reference carries over unchanged. It also used a rounded $8,000 standard deduction instead of $8,953.27. That leaves $105,422 taxable instead of $101,968.14, so its tax is about $200 too high."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It disallowed the $108 IRA deduction, subtracted a $990 SE-tax deduction that doesn't exist because the farm loss makes net SE earnings negative, and added back $2,500 of capital loss. It used an invented phase-out formula giving a $3,957 standard deduction instead of $8,953.27, and taxed the result with the old 4.4% bracket ending at $36,840. It then subtracted $520 of made-up credits."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Its $126,626 AGI double-counts the $14,000 SE partnership income (already inside the $19,350) and subtracts the $1,796 domestic production deduction, which is not an AGI adjustment. It also counted only 4 exemptions. Its stated $4,200 does not follow from its own $109,826 taxable income under any Wisconsin bracket schedule."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It put taxable income at about $125,000 instead of $101,968.14. It treated the standard deduction as almost fully phased out (it is still $8,953.27) and allowed only three $700 dependent exemptions, dropping the two filer exemptions. It then taxed the result at about $6,200."
+us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It replaced the joint schedule with a made-up 4% rate on the first $23,000 and 5.3% on everything above, skipping the 3.5% bracket and the 4.4% band that runs to about $69.3k. It also never subtracted the $3,500 of exemptions."
+us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It took a $25,000 standard deduction without applying Wisconsin's sliding-scale phase-down, which cuts it to $8,953.27 at this income. It also subtracted the $1,796 domestic production deduction from Wisconsin AGI. Taxable income came out at $82,566 instead of $101,968.14."
+us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Its $119,774.69 taxable income is $17,807 above $101,968.14, which matches leaving out the $16,800 farm loss plus a smaller standard deduction. It then taxed that amount using the pre-Act-15 4.4% bracket, which reproduces its $5,829.93."
+us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It subtracted the five $700 personal exemptions ($3,500) dollar-for-dollar from tax as if they were a credit. They are deductions from income, worth only 5.3% of $3,500 at this income. It also used the old 3.54%/4.65% rates and bracket widths."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It used a $13,506 sliding-scale standard deduction instead of the $8,953.27 that applies at $114,421 of Wisconsin AGI, which understated taxable income by about $4,400. It then subtracted $150 of 'dependent credits', but Wisconsin has no such nonrefundable credit here."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no derivation. Its $4,652.12 is $52.50 above the $4,599.62 that $101,968.14 of taxable income produces under the Act 15 joint brackets, which equals roughly $990 of extra income at the 5.3% marginal rate."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It started from Wisconsin AGI of $92,978, $21,443 below the correct $114,421.41 (wages less 401(k) deferral, plus 401(k) distribution and partnership income, less farm loss, $3,000 capital loss and $108 IRA). Taxable income therefore fell to $78,969 instead of $101,968.14."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Its $82,256 taxable income is about $19,700 below the correct $101,968.14 because it understated AGI or overstated deductions. It then taxed that amount with brackets steeper than the Act 15 schedule, so the two errors partly cancel at $4,356."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no derivation. Its $6,950 is $2,350 above the correct $4,599.62, which equals about $44,000 of extra taxable income at 5.3%. That size of gap means it skipped the farm loss, capital loss, sliding-scale standard deduction and exemptions."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It gave no derivation. Its $4,125 is $474.62 below the $4,599.62 that $101,968.14 of taxable income yields under the Act 15 joint brackets with no nonrefundable credits, which equals about $8,950 of taxable income left out at 5.3%."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It gave no derivation. It reports tax after 'nonrefundable credits' of $3,212, $1,387.62 below the correct $4,599.62; no Wisconsin nonrefundable credit applies, and taxable income is $101,968.14."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It gave no derivation. Its $2,490 is $2,109.62 below the $4,599.62 owed on $101,968.14 of taxable income, a gap equal to about $39,800 of taxable income at 5.3% or credits that don't apply. The household has no Wisconsin nonrefundable credits."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"It added a $666.67 Wisconsin early-withdrawal penalty into income tax, which is not part of the tax computed on taxable income. It also applied a $500 capital-loss cap add-back, subtracted medical-insurance premiums that don't qualify, and took a Married Couple Credit even though the spouse has no earnings."
+us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No value or explanation was submitted for state_income_tax_before_refundable_credits, so there is no answer to score against $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"No value or explanation was submitted for state_income_tax_before_refundable_credits, so there is no answer to score against $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It claimed deductions and credits wipe out Wisconsin tax. At $114,421.41 of Wisconsin AGI, the sliding-scale standard deduction is only $8,953.27 and exemptions are $3,500, so $101,968.14 stays taxable and produces $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It assumed deductions and credits exceed the tax. Wisconsin's standard deduction phases down to $8,953.27 at this income, exemptions total $3,500, and there are no nonrefundable credits, so $101,968.14 is taxable and tax is $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It used a $15,166 standard deduction instead of the $8,953.27 sliding-scale amount, counted only two $700 exemptions instead of five, and left out the $3,000 capital loss from income. It then subtracted a $472 itemized-deduction/medical credit, but no Wisconsin nonrefundable credit applies."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It gave no derivation. Its $2,800 is $1,799.62 below the $4,599.62 owed on $101,968.14 of taxable income, which reflects about $34,000 of deductions or credits the household does not have."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $4,872 equals tax on roughly the correct $101,968 taxable income under the pre-Act-15 schedule, where the 4.4% bracket stops near $40k. Under the 2026 schedule the 4.4% band runs to about $69.3k, cutting tax to $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"It applied 'exemption credits', but Wisconsin's $700-per-person exemptions are subtracted from income, not from tax. With $3,500 of exemptions and the $8,953.27 standard deduction, taxable income is $101,968.14 and tax is $4,599.62, not $4,071."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It subtracted a $2,340 health-insurance premium deduction that Wisconsin does not allow for these employer-plan premiums, and added a $2,500 capital-loss add-back. The estimated 2026 brackets and standard deduction it used yield $3,254.36, $1,345 below the $4,599.62 owed on $101,968.14."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It gave no derivation. Its $5,654 is $1,054.38 above the $4,599.62 owed, which equals about $19,900 of extra taxable income at 5.3%, or the old 4.4% bracket combined with a smaller standard deduction or fewer exemptions."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It started from the correct $114,422 AGI but reported $4,200, $399.62 short. The $8,953.27 standard deduction and $3,500 exemptions leave $101,968.14, which the 3.5%/4.4%/5.3% schedule taxes at $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It subtracted a $4,136 health-premium deduction from Wisconsin AGI, bringing it to $110,286. That deduction does not apply, and its larger $10,208 standard deduction also comes from the understated AGI. Taxable income should be $101,968.14 with $4,599.62 of tax."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,"Its $4,800 matches roughly the correct $102k of taxable income taxed with the 4.4% bracket ending near $40k. The 2026 schedule widened by Act 15 extends the 4.4% band to about $69.3k, giving $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It gave no derivation. Its rounded $4,500 is $99.62 below the $4,599.62 owed on $101,968.14 of taxable income, so it did not complete the exact calculation: $8,953.27 standard deduction, $3,500 exemptions, then the 3.5%/4.4%/5.3% schedule."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It never subtracted the five $700 exemptions ($114,422 − $6,500 = $107,900) and used a $6,500 standard deduction instead of $8,953.27, overstating taxable income by about $5,900. It then taxed the result under brackets that stop the 4.4% rate well below the 2026 level of about $69.3k."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Its taxable income of $101,947 is essentially correct, but it split the brackets into equal $29,350 bands, ending the 4.4% rate at $58,700. The 2026 joint schedule applies 3.5% to about $20.1k and 4.4% to about $69.3k, giving $4,599.62 instead of $4,610."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Its taxable income of about $102,274 is close to the correct $101,968.14, but $4,876 comes from the pre-Act-15 schedule, where 4.4% ends near $40k. The 2026 widened 4.4% band to about $69.3k produces $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It applied 2024 bracket widths with wrong rates (4.75% and 5.75%) instead of the 2026 3.5%/4.4%/5.3% joint schedule with the 4.4% band to about $69.3k. It also understated AGI at $106,225, used a $13,500 standard deduction instead of $8,953.27, and left out the $3,500 of exemptions."
+us,scenario_064,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It treated the federal overtime deduction as reducing AGI and then added back $13,899, but federal AGI already includes all wages and equals $114,421.41. After that, its deductions of about $21,600 far exceed the $12,453.27 of sliding-scale standard deduction plus exemptions, so taxable income landed near $90,000 instead of $101,968.14."
+us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was submitted for state_income_tax_before_refundable_credits, so there is no answer to score against $4,599.62."
+us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It applied a 6.27% rate to its third bracket, but Wisconsin's 2026 joint third bracket is 5.3%. Its $116,737 AGI leaves out the $2,315 traditional 401(k) deferral, and its $5,481 standard deduction is below the correct $8,953.27."
+us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It submitted 0 with no derivation, although $114,421.41 of Wisconsin AGI leaves $101,968.14 taxable after the $8,953.27 standard deduction and $3,500 of exemptions, producing $4,599.62 of tax."
+us,scenario_064,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"It used a $19,800 standard deduction instead of the phased-down $8,953.27 and left out the $3,500 of exemptions, putting taxable income at $94,600. It then taxed that under the pre-Act-15 brackets, so the errors partly offset at $4,481."
+us,scenario_064,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"It gave no derivation. Its $4,448 is $151.62 below the $4,599.62 that the 2026 3.5%/4.4%/5.3% joint schedule gives on $101,968.14, which equals about $2,860 of taxable income left out at 5.3%."
+us,scenario_064,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It took a $31,300 standard deduction without the sliding-scale phase-down, which leaves only $8,953.27 at this income, and never subtracted the $3,500 of exemptions. Its $127,253 AGI is about $12,800 too high. It also used invented bracket breaks at $29,430 and $58,860."
+us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It removed the $13,899 FLSA overtime premium from wages, but Wisconsin AGI includes all wages, at $114,421.41. It also used a $13,500 standard deduction instead of $8,953.27, skipped the exemptions, applied made-up 3.5%/4.65% brackets, and then adjusted to an unsupported $4,221.67."
+us,scenario_064,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It subtracted $11,827 of retirement contributions (including Roth amounts and the dependent's contributions) and medical expenses from Wisconsin income. Only the head's $2,315 401(k) deferral and $108 IRA reduce AGI, and Wisconsin has no medical-expense subtraction, so taxable income is $101,968.14 and tax is $4,599.62."
us,scenario_064,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model enumerated every Wisconsin refundable credit — Homestead (income too high), the WI EITC (federal EITC is $0, and the WI credit is a straight percentage of it), Farmland Preservation, and the Veterans and Surviving Spouses Property Tax Credit — and stated that each one does not apply, then submitted $154 rather than the $0 its own enumeration produces. No credit it named yields $154: the Homestead Credit is additionally barred because no rent or property tax is listed, and the Married Couple and Itemized Deduction credits it correctly identified as nonrefundable are excluded from this output entirely."
us,scenario_064,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model omitted the state_refundable_credits key from its outputs object, so it submitted no value for a variable the prompt required exactly once. The correct derivation sums Wisconsin's refundable credits — WI EITC at $0 because the federal EITC is $0 on $97,295 of wages, Homestead at $0 under its household-income ceiling and with no rent or property tax listed, Farmland Preservation at $0 absent a preservation agreement — for a total of $0."
us,scenario_064,state_refundable_credits,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no state_refundable_credits entry, leaving the required key unanswered rather than making a substantive Wisconsin credit error. Working the rule set through yields $0: the WI EITC is a percentage of a federal EITC that is $0 at this income, the Homestead Credit fails both the income ceiling and the requirement that rent or property tax be paid, and the Farmland Preservation Credit requires an unlisted farmland preservation agreement."
@@ -5976,102 +6044,104 @@ us,scenario_067,dependent1_medicare_eligible,kimi-k3,llm_error,age_disability,Fa
us,scenario_067,dependent1_medicare_eligible,muse-spark-1.3,llm_error,age_disability,False,"The model granted Medicare 'via disability' from the disability flag and benefit amount alone. It skipped the requirement of 24 months of SSDI entitlement, which is 0 here by default. The generic disability-benefit input is not SSDI."
us,scenario_067,dependent1_medicare_eligible,muse-wonder,llm_error,age_disability,False,"The model claimed the dependent qualifies through Medicare's disability pathway. That pathway requires at least 24 months receiving SSDI, which is unlisted and defaults to 0, and the listed 'disability benefits' are not SSDI. At age 23 with no ESRD, Dependent 1 is ineligible."
us,scenario_067,dependent1_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model applied 'disability rules' as if a disability flag plus disability benefits created Medicare eligibility. Under-65 Medicare actually requires 24 months of SSDI entitlement or ESRD, and neither is present. The 23-year-old is therefore not eligible."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It deducted the $1,845 auto loan interest as OBBBA vehicle-loan interest, but no fact shows a qualifying new vehicle loan. It also used a $32,600 standard deduction instead of $32,200, which led to $5,347. It then submitted $3,391, a number its own work does not support."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Everything matches the reference except that it subtracted the $1,845 household auto loan interest as a qualified vehicle-loan deduction. No fact shows a new, U.S.-assembled vehicle loan, so that deduction does not apply. This cut taxable income to $53,257 instead of $55,102 and understated tax by $221.40."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $28,500 MFJ standard deduction instead of the 2026 $32,200 and overstated tax on the resulting income at about $7,080. It then subtracted an invented $1,266 of credits instead of the single $500 Credit for Other Dependents."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It assumed no nonrefundable credits, so it never applied the $500 Credit for Other Dependents for the disabled 23-year-old qualifying child. Its $6,900 also overstates even the correct pre-credit tax of $6,116.24 on $55,102 of taxable income."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It reached about $5,620 using roughly correct figures, then 'adjusted' to $6,280 with a $32,300 standard deduction and 2025 bracket edges ($23,850). The correct 2026 figures are a $32,200 deduction and a 10% bracket ending at $24,800, which give $5,616.24 after the $500 ODC."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It made up about $6,500 of mortgage interest from the $100,000 balance, although no interest is listed. It then claimed the roughly $23,121 itemized total beat a $32,300 standard deduction, when it is smaller. After computing $6,722 it submitted an unrelated $5,594 instead of using the $32,200 standard deduction path."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"Its own work reached the correct $6,116 pre-credit tax and about $5,616 after the $500 ODC. It then dropped that result for an unexplained 'bracket estimate' of $6,104."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $1,845 auto loan interest as a deduction even though nothing shows a qualified new-vehicle loan. That lowered taxable income to $53,257 instead of $55,102 and understated tax by $221.40."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $30,000 MFJ standard deduction, the pre-OBBBA 2025 figure, instead of the 2026 $32,200. It also used estimated 10% bracket edges of $23,850 to $24,500 instead of $24,800. Taxable income came out at $57,302 instead of $55,102."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed about $5,568 with a $32,600 standard deduction and the $500 ODC, then submitted $10,500 citing 'self-employment-like adjustments' that do not exist here. There is no self-employment income, and the correct result is $6,116.24 minus $500."
-us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the dependent's $10,800 of disability benefits to the joint AGI, but they are the dependent's own nontaxable benefits. It used a $30,600 standard deduction instead of $32,200. It also denied the $500 Credit for Other Dependents, which a permanently disabled adult qualifying child gets regardless of age."
-us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed TCJA expired in 2026 and used a $16,500 standard deduction, three personal exemptions and a 15% bracket. OBBBA made TCJA permanent, so 2026 has a $32,200 standard deduction, no exemptions, 10%/12% brackets and the $500 ODC."
-us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 rules: itemized medical plus Indiana tax, three $5,300 exemptions, a 15% bracket and no credits. Under OBBBA, the $32,200 standard deduction is larger than its $19,108 itemized total, exemptions stay at zero, and the $500 ODC applies."
-us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its $5,486.84 matches a taxable income of $53,957: it took the 2025 $31,500 standard deduction and also deducted the $1,845 auto loan interest, then used a $24,400 10% bracket. The correct calculation uses the 2026 $32,200 deduction, no vehicle-interest deduction and a $24,800 10% bracket."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed TCJA sunset and used personal exemptions, a 15% bracket and itemizing against a $15,900 standard deduction. It also wrongly put 85% of the dependent's disability benefits on the parents' return as taxable Social Security. Finally, it skipped the $500 ODC that applies under permanent TCJA rules."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It said there were no dependents, so it skipped the $500 Credit for Other Dependents for the disabled adult qualifying child. It also used a $31,200 standard deduction and miscalculated taxable income as about $60,400 instead of $55,102."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed TCJA expired in 2026 and used itemized deductions plus $16,050 of personal exemptions and the 15% bracket. Under OBBBA, the $32,200 standard deduction, the 10%/12% brackets and the $500 ODC give $5,616.24."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used pre-TCJA 2026 rules: personal exemptions and the 15% bracket. It also made up $4,500 of mortgage interest even though only a loan balance is listed. Under current law the $32,200 standard deduction applies, and the $500 ODC reduces the $6,116.24 tax."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"The correct path is $87,302 AGI less the $32,200 standard deduction, giving $55,102 taxable income, $6,116.24 tax and $5,616.24 after the $500 ODC. Its $7,534 is about $1,900 higher. That matches adding the dependent's $10,800 disability benefits to taxable income and skipping the $500 ODC."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction instead of the 2026 MFJ $32,200. It also used the 2025 10% bracket edge of $23,850 instead of $24,800, which overstated taxable income and tax."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $6,412.24 pre-credit tax matches $57,302 of taxable income: it used the pre-OBBBA $30,000 standard deduction and the 2024 $23,200 10% bracket edge. The 2026 values are $32,200 and $24,800."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction and a $23,850 10% bracket edge, both pre-2026 values, instead of the 2026 MFJ $32,200 and $24,800. That gave $57,302 of taxable income instead of $55,102."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"Its $5,394.84 is exactly the reference minus 12% of $1,845. That means it deducted the household auto loan interest as qualified vehicle-loan interest, which the facts do not support."
-us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction and a $23,600 10% bracket edge instead of the 2026 MFJ $32,200 and $24,800. That overstated taxable income at $57,302 and pre-credit tax."
-us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It denied the $500 Credit for Other Dependents by testing the 23-year-old under the qualifying-relative gross income limit. A permanently and totally disabled child is a qualifying child with no age limit and no gross income test. It also used a $25,200 10% bracket edge instead of $24,800."
-us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The correct calculation is $55,102 of taxable income after the $32,200 standard deduction, $6,116.24 of tax and $5,616.24 after the $500 ODC. Its $4,567 is about $1,050 too low. That matches roughly $8,700 of extra deductions or credits that the facts do not support."
-us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It cited mortgage and medical itemized deductions, but no mortgage interest is listed and net medical is only about $16,621. The $32,200 standard deduction therefore applies. Its $4,940 understates the $6,116.24 tax less the $500 ODC."
-us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It added $9,180 (85%) of the dependent's $10,800 disability benefits to the joint return as taxable Social Security. Those benefits belong to the dependent and are not in the parents' gross income, so taxable income is $55,102, not $64,282."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The correct calculation is $87,302 less the $32,200 standard deduction, giving $55,102, then $6,116.24 of tax and $5,616.24 after the $500 ODC. Its $8,500 exceeds even the pre-credit tax. That matches stale deduction or bracket values with no $500 Credit for Other Dependents."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed TCJA expired and used itemized deductions, $15,750 of personal exemptions and the 15% bracket with no credits. OBBBA kept the $32,200 standard deduction, the 10%/12% brackets and the $500 ODC."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a TCJA-sunset regime: a $16,600 standard deduction, three $5,300 exemptions and a 15% bracket. The permanent-TCJA 2026 rules have a $32,200 standard deduction, no exemptions, 10%/12% rates and the $500 ODC."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It assumed TCJA sunset, so it used personal exemptions, the 15% bracket and said the other-dependent credit had expired. OBBBA made the $32,200 standard deduction and the $500 ODC permanent for 2026."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used an inflation-adjusted pre-TCJA $17,291 standard deduction and a 15% bracket, and gave no ODC. Under 2026 current law the standard deduction is $32,200 and the rates are 10%/12%, with the $500 ODC for the disabled adult child."
-us,scenario_067,federal_income_tax_before_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It claimed no dependents, so it skipped the $500 Credit for Other Dependents for the permanently disabled 23-year-old qualifying child. It also used the 2025 $31,500 standard deduction instead of the 2026 $32,200."
-us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction and a $24,400 10% bracket edge instead of the 2026 MFJ $32,200 and $24,800. That overstated taxable income at $56,502 instead of $55,102."
-us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It wrongly added $9,180 of the dependent's disability benefits to the joint AGI as taxable Social Security. It also deducted the $1,845 auto loan interest, though no qualifying vehicle loan is shown, so taxable income came out at $62,437 instead of $55,102."
-us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It applied a $2,000 Child Tax Credit for the 23-year-old, but the CTC requires the child to be under 17. A disabled adult qualifying child gets only the $500 Credit for Other Dependents. It also added the dependent's disability benefits to AGI and used a $30,000 standard deduction."
-us,scenario_067,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,other,False,"Its method matches the reference, but it rounded pre-credit tax to $6,122. The exact figure is $2,480 + 12% × $30,302 = $6,116.24, which leaves $5,616.24 after the $500 ODC."
-us,scenario_067,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,other,False,"It rounded taxable income to $55,100 and miscalculated the tax as $6,124. The 10%/12% brackets on $55,102 give exactly $6,116.24, and $5,616.24 after the $500 ODC."
-us,scenario_067,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It computed the correct $6,116.24 tax but applied no nonrefundable credits, reasoning that a 23-year-old cannot get the CTC. It missed that a permanently disabled adult qualifying child qualifies for the $500 Credit for Other Dependents."
-us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It treated the dependent's $10,800 of disability benefits as taxable income on the joint return, raising AGI to $98,102 instead of $87,302. It also used a $31,200 standard deduction and a $24,150 10% bracket edge instead of $32,200 and $24,800."
-us,scenario_067,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It applied a made-up $10,000 Child Tax Credit and wiped out the whole liability. No child under 17 is present; the 23-year-old disabled dependent gets only the $500 Credit for Other Dependents against $6,116.24 of tax."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its reasoning used a $32,600 standard deduction instead of $32,200 and subtracted the $1,845 auto loan interest, which the reference does not allow because no facts show a qualifying vehicle. That produced $5,347. It then submitted $3,391, which matches none of its own arithmetic; the correct path ($55,102 taxable, $6,116.24 tax, minus the $500 credit for other dependents) gives $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It took the OBBBA vehicle loan interest deduction for the $1,845 of auto loan interest, but no household fact shows a new, U.S.-assembled vehicle bought after 2024, so the reference allows no deduction. That cut taxable income from $55,102 to $53,257 and lowered tax by exactly 12% × $1,845 = $221.40, from $5,616.24 to $5,394.84."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It subtracted the $1,845 of auto loan interest as a vehicle loan interest deduction that the reference does not allow, since no facts show a qualifying new, U.S.-assembled vehicle. Taxable income should be $55,102, not $53,257. Everything else matches, so the answer is $221.40 (12% × $1,845) too low."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $28,500 standard deduction instead of the 2026 MFJ figure of $32,200, which inflated taxable income to $58,802. It then subtracted an invented $1,266 of nonrefundable credits ('Disability Exemption Credit') instead of the actual $500 credit for other dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It explicitly assumed no nonrefundable credits, missing the $500 credit for other dependents for the disabled 23-year-old. Its $6,900 is also above the correct pre-credit tax of $6,116.24 ($55,102 after the $32,200 standard deduction, taxed at 10%/12% with the 10% bracket ending at $24,800), so it used a smaller deduction or wrong bracket thresholds."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its own derivation reached about $5,620. It then 'adjusted' to 2025 bracket edges ($23,850) and a $32,300 standard deduction and submitted $6,280, which those figures do not even support. The 2026 MFJ values are a $32,200 standard deduction and a 10% bracket ending at $24,800, giving $6,116.24 − $500 = $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It imputed $6,500 of mortgage interest from the $100,000 balance, but no interest amount is listed, so it counts as $0. It also claimed $23,121 of itemized deductions exceeded the roughly $32,300 standard deduction, when it is smaller. The $32,200 standard deduction applies, giving $5,616.24, but the model submitted an unexplained $5,594."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly computed $55,102 of taxable income, $6,116 of tax and about $5,616 after the $500 credit for other dependents. It then dropped that result for an unexplained 'bracket estimate' of $6,104, which matches no consistent bracket computation."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It deducted the $1,845 of auto loan interest on top of the $32,200 standard deduction, but the facts show no qualifying vehicle, so the reference allows no auto loan interest deduction. Taxable income is $55,102, not $53,257, and the answer comes out $221.40 too low."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It projected a $30,000 MFJ standard deduction and a 10% bracket ending at $24,500. The 2026 values are a $32,200 standard deduction and a 10% bracket ending at $24,800. Its taxable income of $57,302 instead of $55,102 produced $5,886 rather than $5,616.24, though the $500 credit for other dependents was handled correctly."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It computed $6,068 of tax using a $32,600 standard deduction (the correct figure is $32,200) and acknowledged the $500 credit for other dependents. It then submitted $10,500 citing nonexistent 'self-employment-like adjustments' and benefit phase-outs, which have nothing to do with this wage-and-pension household."
+us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the dependent's $10,800 of disability benefits to the filers' AGI ($98,102 instead of $87,302) and used a $30,600 standard deduction instead of $32,200. It also denied the $500 credit for other dependents, which applies because a permanently disabled child is a qualifying child at any age."
+us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the pre-TCJA sunset structure: a $16,500 standard deduction, three personal exemptions and a 15% second bracket. OBBBA made the TCJA structure permanent, so 2026 has a $32,200 MFJ standard deduction, no exemptions and a 12% bracket. It also left out the $500 credit for other dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used pre-TCJA law: itemizing about $19,108 of medical expenses and state tax, three $5,300 personal exemptions and a 15% bracket. Under OBBBA's permanent rules, the $32,200 standard deduction beats itemizing, exemptions are zero and the rate is 12%. It also omitted the $500 credit for other dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"By its own statement it subtracted an auto loan interest deduction, which the reference does not allow because the facts show no qualifying vehicle. Using only the $32,200 standard deduction gives $55,102 of taxable income, $6,116.24 of tax and $5,616.24 after the $500 credit, not $5,486.84."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset (personal exemptions, 15% bracket, low standard deduction), but OBBBA keeps a $32,200 MFJ standard deduction and a 12% bracket for 2026. It also added $9,180 of the dependent's disability benefits to the filers' AGI as taxable Social Security, though that income is not part of their return. It omitted the $500 credit for other dependents as well."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It reported $60,400 of taxable income even though $87,302 minus its own $31,200 deduction is $56,102; the correct figure is $55,102 using the $32,200 deduction. It also wrongly treated the household as having no credits, missing the $500 credit for other dependents for the disabled 23-year-old."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It computed tax under an assumed TCJA expiration, with $19,064 itemized, $16,050 of personal exemptions and a 15% bracket. OBBBA made TCJA permanent, so 2026 uses the $32,200 standard deduction, no exemptions and 10%/12% rates. It also never applied the $500 credit for other dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used pre-TCJA 2026 rules with personal exemptions and a 15% bracket, and inflated itemized deductions with $4,500 of invented mortgage interest even though no interest amount is given. Current law gives a $32,200 standard deduction, $55,102 of taxable income and $6,116.24 of tax less the $500 credit for other dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"Its $7,534 is far above the correct pre-credit tax of $6,116.24 ($87,302 − $32,200 = $55,102, taxed at 10% up to $24,800 and 12% above). That means it used a smaller standard deduction or wrong brackets, and it also skipped the $500 credit for other dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction and the 2025 10% bracket edge of $23,850 instead of the 2026 figures of $32,200 and $24,800. That overstated taxable income ($56,502 vs. $55,102) and tax ($6,303.24 vs. $6,116.24) before the correctly applied $500 credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $5,912.24 corresponds to a $30,000 standard deduction ($57,302 taxable) and the 2024 10% bracket edge of $23,200 ($2,320 + 12% × $34,102 − $500). The 2026 MFJ standard deduction is $32,200 and the 10% bracket ends at $24,800."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the old $30,000 MFJ standard deduction and the 2025 10% bracket edge of $23,850 instead of the 2026 values of $32,200 and $24,800. Its taxable income was $57,302 instead of $55,102 and its pre-credit tax $6,399.24 instead of $6,116.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"Its $5,394.84 equals the reference minus exactly 12% × $1,845, so it deducted the auto loan interest, which the reference does not allow because no facts show a qualifying vehicle. Taxable income should be $55,102 after only the $32,200 standard deduction."
+us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $30,000 standard deduction and a $23,600 10% bracket edge instead of the 2026 MFJ figures of $32,200 and $24,800. Taxable income came out $2,200 too high ($57,302 vs. $55,102)."
+us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,categorical_eligibility,False,"It denied the $500 credit for other dependents by applying the qualifying-relative gross income test. A permanently and totally disabled child is a qualifying child with no age limit and no gross income test, so the credit applies. It also put the 10% bracket edge at $25,200 instead of $24,800."
+us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave no computation, and $4,567 is about $1,050 below the correct result. The correct path is $87,302 of AGI less the $32,200 standard deduction = $55,102 taxable, $6,116.24 at 10%/12%, minus the $500 credit for other dependents = $5,616.24. Its figure implies an extra deduction or credit the facts don't support."
+us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It says it applied itemized deductions using the mortgage and medical facts. But no mortgage interest is listed, and medical expenses above the 7.5% floor ($16,621) are below the $32,200 standard deduction, so the standard deduction applies. The correct liability is $5,616.24, not $4,940."
+us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It added $9,180 (85% of the dependent's $10,800 disability benefits) to the joint return as taxable income. Those benefits belong to the dependent and are not in the filers' AGI. Removing them gives $55,102 of taxable income and $5,616.24 instead of $6,717.84; the rest of its computation was correct."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"Its unexplained $8,500 is well above the correct figure. AGI of $87,302 less the $32,200 standard deduction gives $55,102 of taxable income and $6,116.24 at 10%/12% rates. Subtracting the $500 credit for other dependents gives $5,616.24, so the model overstated taxable income or the rates."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied post-sunset rules: itemized deductions, $15,750 of personal exemptions, a 15% bracket and no credits. OBBBA keeps the $32,200 MFJ standard deduction, zero exemptions, the 12% bracket and the $500 credit for other dependents for 2026."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, with a $16,600 standard deduction, three $5,300 exemptions and a 15% bracket. 2026 law gives a $32,200 standard deduction, no exemptions and a 12% bracket. It also omitted the $500 credit for other dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It applied post-TCJA-sunset rules (personal exemptions, 15% bracket) and claimed the credit for other dependents had expired. OBBBA made both the $32,200 standard deduction structure and the $500 credit for other dependents permanent, giving $6,116.24 − $500 = $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used an inflation-adjusted pre-TCJA $17,291 standard deduction and a 15% bracket, and gave no credit. Current 2026 law gives a $32,200 MFJ standard deduction, a 12% second bracket and the $500 credit for other dependents for the disabled adult child."
+us,scenario_067,federal_income_tax_before_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It explicitly claimed no dependents, missing that the permanently disabled 23-year-old is a qualifying child at any age and earns the $500 credit for other dependents. It also used a $31,500 standard deduction instead of $32,200."
+us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $30,800 standard deduction and a $24,400 10% bracket edge instead of the 2026 MFJ values of $32,200 and $24,800. That overstated taxable income by $1,400 and tax by $176 before the correctly applied $500 credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It added $9,180 of the dependent's disability benefits to the joint AGI as taxable Social Security, though that income is not part of the filers' return. It also subtracted $1,845 of auto loan interest, which the reference does not allow. Correct taxable income is $87,302 − $32,200 = $55,102."
+us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It applied the $2,000 child tax credit to the 23-year-old, but the CTC requires a child under 17, so only the $500 credit for other dependents applies. It also added the dependent's $10,800 of benefits to AGI (and summed them wrong) and used the outdated $30,000 standard deduction and $23,850 bracket edge."
+us,scenario_067,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,other,False,"It had the right structure ($55,102 taxable, $500 credit for other dependents) but approximated pre-credit tax as about $6,122. The exact figure is 10% × $24,800 + 12% × $30,302 = $6,116.24, giving $5,616.24, not $5,622."
+us,scenario_067,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,other,False,"It rounded taxable income to about $55,100 and tax to about $6,124 instead of computing exactly $2,480 + 12% × $30,302 = $6,116.24. Its answer ended $7.76 above the exact $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"It computed the correct pre-credit tax of $6,116.24 but applied no nonrefundable credits, reasoning that a 23-year-old cannot get the CTC. It missed that the permanently disabled adult child is still a qualifying child for the $500 credit for other dependents, which brings the result to $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It treated the dependent's $10,800 of disability benefits as taxable income on the joint return, raising AGI to $98,102 instead of $87,302. It also used a $31,200 standard deduction instead of $32,200, overstating taxable income by about $11,800."
+us,scenario_067,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"It invented a $10,000 child tax credit that wiped out the whole liability. The only dependent is 23, so the CTC (under-17 rule) does not apply, and the only credit is the $500 credit for other dependents. That leaves $5,616.24, not $0."
us,scenario_067,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the 23-year-old disabled dependent as a qualifying child for the refundable Additional Child Tax Credit. Disability can remove the age limit for the qualifying-child relationship test used for dependency and EITC purposes, but it does not override the CTC requirement that the child be under age 17, so the ACTC is $0."
us,scenario_067,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented five qualifying children even though the household contains only one dependent, age 23, and then constructed a $10,000 CTC and $1,210 ACTC from that nonexistent child count. The sole dependent fails the CTC under-age-17 requirement, so there is no CTC remainder to refund and no ACTC."
us,scenario_067,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 60 as above the Medicare eligibility threshold. The applicable age threshold is 65, and no disability-based Medicare pathway or other exception applies to the head, so the correct value is 0."
-us,scenario_067,local_income_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"Treated the prompt's four named cities as the complete list for this output, so it left out Indiana county income tax. The engine includes that tax: 1.6% (Adams County) × $84,302 Indiana AGI after $3,000 of exemptions = $1,348.83."
-us,scenario_067,local_income_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"Read NYC/Philadelphia/KC/St. Louis as the only taxes in this output and answered $0. It missed that PolicyEngine includes Indiana county income tax here, which is 1.6% of $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"Said Indiana has no local income tax at all, which is false: every Indiana county levies a county income tax on residents. PolicyEngine reports that tax here, 1.6% of $84,302 Indiana AGI after exemptions = $1,348.83."
-us,scenario_067,local_income_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Correctly counted Indiana county income tax, but $1,800 implies a rate of about 2.1% on roughly $84–87k. PolicyEngine applies the default Adams County rate of 1.6% to Indiana AGI of $84,302 ($87,302 minus $3,000 of $1,000-per-person exemptions), giving $1,348.83."
-us,scenario_067,local_income_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"Read the four named cities as the complete scope of the output and left out Indiana county income tax. The engine computes that tax as 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"Stated that the output covers only NYC/Philadelphia/KC/St. Louis taxes, so it answered $0. PolicyEngine's local_income_tax also includes the Indiana county tax, which is 1.6% of Indiana AGI after exemptions ($84,302) = $1,348.83."
-us,scenario_067,local_income_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Explicitly ruled Indiana county income tax out of this output because of the four-city list. PolicyEngine counts it here: Adams County 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"Checked only the four named city taxes and never looked at Indiana county income tax. That tax produces $1,348.83 (1.6% of $84,302 Indiana AGI)."
-us,scenario_067,local_income_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"Read the listed jurisdictions as the full set covered by the output, so it left out Indiana's county income tax. PolicyEngine applies that tax at 1.6% to $84,302, giving $1,348.83."
-us,scenario_067,local_income_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"Assumed only the four named city taxes count and answered $0. The engine includes the Indiana county tax: 1.6% × ($87,302 − $3,000 exemptions) = $1,348.83."
-us,scenario_067,local_income_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"Read the output as limited to the four listed city taxes, so it left out Indiana county income tax. That tax is 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"Read the four named taxes as the complete list and answered $0. It missed the Indiana county income tax PolicyEngine includes, which is 1.6% of $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Checked only NYC/Philadelphia/KC/St. Louis and left out Indiana county income tax. The engine computes that tax as 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"Read the prompt's city list as complete, so it never applied Indiana county income tax. That tax is 1.6% of Indiana AGI after $3,000 of exemptions = $1,348.83."
-us,scenario_067,local_income_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"Its $0 answer matches reading the four-city list as the complete scope and leaving out Indiana county income tax. The engine computes that tax as 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"Limited the output to the four named city taxes and left out Indiana county income tax. That tax gives $1,348.83 at Adams County's 1.6% rate on $84,302."
-us,scenario_067,local_income_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"Said living in Indiana triggers no local income tax, which is false: Indiana counties levy a county income tax on residents. PolicyEngine reports it here as 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Explicitly excluded Indiana county taxes from this output because of the four-city list. PolicyEngine includes the county tax in local_income_tax: 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"Applied only the four named city taxes and answered $0. It missed Indiana county income tax, which comes to $1,348.83 (1.6% of $84,302 Indiana AGI)."
-us,scenario_067,local_income_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"Assumed Indiana's local tax is not the kind of tax this output covers. PolicyEngine reports Indiana county income tax here: 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"Read the listed cities as the only jurisdictions with tax in this output, so it left out Indiana county income tax. That tax is 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Concluded the household lives in no jurisdiction with a separate local income tax because no city was listed. PolicyEngine applies Indiana county income tax using Adams County's 1.6% rate on $84,302, giving $1,348.83."
-us,scenario_067,local_income_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"Asserted that PolicyEngine's local income tax covers only NYC/Philadelphia/KC/St. Louis. The engine also adds Indiana county tax to this output, which is 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"Checked only city-level taxes and left out Indiana's county income tax. PolicyEngine includes that tax here at 1.6% of $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"Read the four specified taxes as complete and answered $0. It missed Indiana county income tax, which PolicyEngine computes as 1.6% × ($87,302 − $3,000) = $1,348.83."
-us,scenario_067,local_income_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"Treated the four listed taxes as the only ones counted, so it left out Indiana county income tax. That tax is $1,348.83 at 1.6% of $84,302."
-us,scenario_067,local_income_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Applied no tax because no local jurisdiction was named. PolicyEngine falls back to Adams County's 1.6% Indiana county tax on $84,302 Indiana AGI, which gives $1,348.83."
-us,scenario_067,local_income_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"Limited the output to NYC/Philadelphia/KC/St. Louis and left out Indiana county income tax. That tax is 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"Read the four named city taxes as the full scope of the output and answered $0. It missed the Indiana county tax PolicyEngine includes, which is 1.6% of $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"Treated the listed cities as the only taxing jurisdictions for this output, so it left out Indiana county income tax. That tax is 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"Read the four requested city taxes as the complete scope and answered $0. PolicyEngine adds Indiana county income tax here: 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Concluded that Indiana is outside the listed jurisdictions, so it applied no tax. The engine computes Indiana county income tax as 1.6% of $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"Treated Indiana as outside the output's coverage because of the four-city list. PolicyEngine includes Indiana county tax in local_income_tax: 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"Read the listed categories as complete and found no Indiana match. It missed the Indiana county income tax PolicyEngine computes, $1,348.83 at 1.6% of $84,302."
-us,scenario_067,local_income_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"Treated Indiana as outside the jurisdictions included in the output and answered $0. PolicyEngine includes Indiana county income tax, which is 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"Limited the output to the named city taxes and left out Indiana county income tax. That tax is 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,gpt-6.1-terra,prompt_ambiguity,state_local_rule,False,"Explicitly ruled Indiana county taxes out of this output. PolicyEngine puts the county tax in local_income_tax: Adams County 1.6% × $84,302 Indiana AGI = $1,348.83."
-us,scenario_067,local_income_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"Said no local income taxes apply in Indiana, which is false: all Indiana counties levy a county income tax on residents. PolicyEngine reports it here as 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"Treated the four named cities as the only sources of this output and answered $0. It missed Indiana county income tax, which is 1.6% of $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"Explicitly excluded Indiana county tax because of the four-city list. PolicyEngine includes it: 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"Read the four named cities as the complete scope and answered $0. It missed Indiana county income tax, which comes to $1,348.83 at 1.6% of $84,302."
-us,scenario_067,local_income_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"Recognized that Indiana has county income taxes but excluded them because of the city list. PolicyEngine includes the county tax in this output: 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,inkling,prompt_ambiguity,state_local_rule,False,"Checked only the four listed city taxes and left out Indiana county income tax. That tax is $1,348.83 (1.6% of $84,302)."
-us,scenario_067,local_income_tax,kimi-k2.6,prompt_ambiguity,state_local_rule,False,"Treated the four named taxes as the only ones this output captures and answered $0. PolicyEngine includes Indiana county income tax here: 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"Applied only the four named city taxes and left out Indiana county income tax. That tax is 1.6% × ($87,302 − $3,000 exemptions) = $1,348.83."
-us,scenario_067,local_income_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"Recognized Indiana's local income tax but defaulted to $0 because no county was given. PolicyEngine uses Adams County when none is specified and applies its 1.6% rate to $84,302 Indiana AGI, giving $1,348.83."
-us,scenario_067,local_income_tax,muse-spark-1.3,prompt_ambiguity,state_local_rule,False,"Said only NYC/Philadelphia/KC/St. Louis count, so it left out Indiana county income tax. That tax is 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"Read the four named cities as the complete scope and answered $0. It missed Indiana county income tax, which is 1.6% of $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"Explicitly ruled county-level Indiana taxes out of this output. PolicyEngine includes them: 1.6% Adams County rate × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"Treated the four city taxes as the only ones modeled and answered $0. PolicyEngine also models Indiana county income tax and reports it here: 1.6% × $84,302 = $1,348.83."
-us,scenario_067,local_income_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"Said local taxes apply only in the listed cities, so it left out Indiana county income tax. That tax is $1,348.83 at 1.6% of $84,302 Indiana AGI."
+us,scenario_067,local_income_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"Treated the four named cities (NYC, Philadelphia, KC, St. Louis) as a complete list and returned $0. PolicyEngine's local_income_tax also includes Indiana county income tax: the default Adams County rate of 1.6% on Indiana AGI of $84,302 gives $1,348.83."
+us,scenario_067,local_income_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"Applied only the four listed city taxes and left out the Indiana county income tax that PolicyEngine reports in local_income_tax. Adams County's 1.6% on Indiana AGI of $84,302 ($87,302 minus $3,000 of exemptions) is $1,348.83."
+us,scenario_067,local_income_tax,claude-fable-5.2,prompt_ambiguity,state_local_rule,False,"Recognized the Indiana county tax but left it out because the prompt's definition lists only four cities. PolicyEngine counts the county tax in local_income_tax: 1.6% (default Adams County) × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"Said Indiana has no local income tax system. In fact every Indiana county levies an income tax on Indiana AGI, and PolicyEngine reports it in local_income_tax. Adams County's 1.6% on $84,302 of Indiana AGI ($87,302 minus three $1,000 exemptions) is $1,348.83."
+us,scenario_067,local_income_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Correctly included Indiana county income tax but gave $1,800 with no stated rate or base, which works out to about 2.1% of Indiana AGI. PolicyEngine applies the default Adams County rate of 1.6% to Indiana AGI of $84,302 (federal AGI $87,302 minus $3,000 of exemptions), giving $1,348.83."
+us,scenario_067,local_income_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"Read the prompt's four-city list as complete and returned $0, missing the Indiana county income tax that PolicyEngine includes in local_income_tax. That tax is 1.6% (Adams County default) × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"Took the output to cover only the NYC, Philadelphia, KC and St. Louis taxes, but PolicyEngine's local_income_tax also includes Indiana county tax. The default Adams County rate of 1.6% on Indiana AGI of $84,302 gives $1,348.83."
+us,scenario_067,local_income_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Explicitly left Indiana county income tax out of the output based on the four-city definition. PolicyEngine includes it: Adams County's 1.6% on $84,302 Indiana AGI ($87,302 minus three $1,000 exemptions) is $1,348.83."
+us,scenario_067,local_income_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"Checked only the four named city taxes and never applied Indiana county income tax, which PolicyEngine reports in local_income_tax. Adams County's 1.6% on $84,302 of Indiana AGI is $1,348.83."
+us,scenario_067,local_income_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"Treated the listed jurisdictions as the only local taxes tracked and returned $0. PolicyEngine's local_income_tax also carries Indiana county tax: 1.6% (Adams County default) × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"Limited the output to the four named city taxes and left out the Indiana county income tax. PolicyEngine computes that tax at the default Adams County rate of 1.6% on Indiana AGI of $84,302, for $1,348.83."
+us,scenario_067,local_income_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"Concluded that Indiana has nothing in the listed local-tax categories, but PolicyEngine's local_income_tax includes Indiana county income tax. That is 1.6% × $84,302 Indiana AGI ($87,302 minus $3,000 of exemptions) = $1,348.83."
+us,scenario_067,local_income_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"Read the four 'covered' city taxes as the whole output and returned $0, skipping the Indiana county income tax that PolicyEngine includes. The default Adams County rate of 1.6% on $84,302 of Indiana AGI gives $1,348.83."
+us,scenario_067,local_income_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Checked only NYC, Philadelphia, KC and St. Louis and missed the Indiana county income tax counted in PolicyEngine's local_income_tax. That tax is 1.6% (Adams County) × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"Treated the four named city taxes as complete and never applied Indiana county income tax. PolicyEngine taxes Indiana AGI of $84,302 ($87,302 minus three $1,000 exemptions) at Adams County's 1.6%, giving $1,348.83."
+us,scenario_067,local_income_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"Found no applicable local tax because it relied on the prompt's four-city definition, but PolicyEngine's local_income_tax includes Indiana county income tax. The default Adams County rate of 1.6% on $84,302 of Indiana AGI gives $1,348.83."
+us,scenario_067,local_income_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"Read the request as covering only NYC, Philadelphia, KC and St. Louis and returned $0. PolicyEngine also includes Indiana county tax in this output: 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"Said living in Indiana does not trigger local income tax, but every Indiana county levies an income tax on Indiana AGI and PolicyEngine reports it as local_income_tax. Adams County's 1.6% on $84,302 ($87,302 AGI minus $3,000 of exemptions) is $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Explicitly left Indiana county taxes out of this variable, but PolicyEngine's local_income_tax includes them. The default Adams County rate of 1.6% on $84,302 of Indiana AGI gives $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"Considered only the four specified city taxes and returned $0, missing the Indiana county income tax included in PolicyEngine's local_income_tax. That tax is 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"Concluded that Indiana has no local tax 'of the type modeled', but PolicyEngine's local_income_tax includes Indiana county income tax. Adams County's 1.6% on Indiana AGI of $84,302 is $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"Required the household to be in one of the four listed cities and returned $0, but PolicyEngine also applies Indiana county income tax. The default Adams County rate of 1.6% on $84,302 of Indiana AGI gives $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Did not count Indiana's county income tax as a separate local income tax, although PolicyEngine reports it in local_income_tax. Adams County's 1.6% on Indiana AGI of $84,302 ($87,302 minus $3,000 of exemptions) is $1,348.83."
+us,scenario_067,local_income_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"Asserted that PolicyEngine's local tax covers only NYC, Philadelphia, KC and St. Louis, but the variable also includes Indiana county income tax. That tax is 1.6% (Adams County default) × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"Looked only for modeled city taxes and missed that PolicyEngine's local_income_tax includes Indiana county income tax. The default Adams County rate of 1.6% on $84,302 of Indiana AGI gives $1,348.83."
+us,scenario_067,local_income_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"Treated the four specified city taxes as a complete list and returned $0. PolicyEngine includes Indiana county income tax in this output: 1.6% × $84,302 Indiana AGI ($87,302 minus three $1,000 exemptions) = $1,348.83."
+us,scenario_067,local_income_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"Read the output as counting only NYC, Philadelphia, KC and St. Louis taxes and missed the Indiana county income tax that PolicyEngine includes. Adams County's 1.6% on $84,302 of Indiana AGI is $1,348.83."
+us,scenario_067,local_income_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Applied no local tax because no jurisdiction was named, but PolicyEngine assigns Indiana households a default county (Adams, 1.6%) and taxes Indiana AGI of $84,302. That gives $1,348.83."
+us,scenario_067,local_income_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"Left out all local tax because none of the four named cities applied, missing the Indiana county income tax in PolicyEngine's local_income_tax. That tax is 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"Treated the NYC, Philadelphia, KC and St. Louis taxes as the whole output and returned $0. PolicyEngine also counts Indiana county tax: the default Adams County rate of 1.6% on $84,302 of Indiana AGI gives $1,348.83."
+us,scenario_067,local_income_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"Checked only the four named city jurisdictions and never applied Indiana county income tax, which PolicyEngine includes in local_income_tax. That tax is 1.6% × $84,302 Indiana AGI ($87,302 minus $3,000 of exemptions) = $1,348.83."
+us,scenario_067,local_income_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"Read the separately requested taxes as only the four city taxes and returned $0, leaving out the Indiana county tax that PolicyEngine reports. Adams County's 1.6% on $84,302 of Indiana AGI is $1,348.83."
+us,scenario_067,local_income_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Concluded that Indiana is not a covered jurisdiction, but PolicyEngine's local_income_tax includes Indiana county income tax. That tax is the default Adams County rate of 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"Treated Indiana as outside the output's coverage based on the four-city definition, but PolicyEngine includes Indiana county income tax in local_income_tax. That tax is 1.6% on $84,302 of Indiana AGI, or $1,348.83."
+us,scenario_067,local_income_tax,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"Applied $0 because no listed category or jurisdiction applied. PolicyEngine taxes Indiana AGI of $84,302 ($87,302 minus three $1,000 exemptions) at the default Adams County rate of 1.6%, giving $1,348.83."
+us,scenario_067,local_income_tax,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"Read the four named jurisdictions as the complete list and left out the Indiana county income tax that PolicyEngine includes. That tax is 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"Treated the output as covering only the specified city taxes, but PolicyEngine's local_income_tax also includes Indiana county income tax. Adams County's 1.6% on $84,302 of Indiana AGI is $1,348.83."
+us,scenario_067,local_income_tax,gpt-6.1-terra,prompt_ambiguity,state_local_rule,False,"Explicitly put Indiana county taxes outside this output, but PolicyEngine includes them in local_income_tax. The default Adams County rate of 1.6% on $84,302 of Indiana AGI ($87,302 minus $3,000 of exemptions) gives $1,348.83."
+us,scenario_067,local_income_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"Said no local income taxes apply in Indiana, but every Indiana county levies an income tax on Indiana AGI and PolicyEngine reports it in local_income_tax. Adams County's 1.6% on $84,302 of Indiana AGI is $1,348.83."
+us,scenario_067,local_income_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"Treated only the four named cities as producing this output and returned $0, missing Indiana county income tax. PolicyEngine computes that tax at 1.6% (Adams County default) × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"Recognized Indiana county tax but put it outside the specified outputs, while PolicyEngine includes it in local_income_tax. Adams County's 1.6% on $84,302 of Indiana AGI is $1,348.83."
+us,scenario_067,local_income_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"Read the four-city list as complete and returned $0, missing the Indiana county income tax that PolicyEngine reports in this variable. That tax is 1.6% × $84,302 Indiana AGI ($87,302 minus $3,000 of exemptions) = $1,348.83."
+us,scenario_067,local_income_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"Acknowledged that Indiana has county taxes but left them out because the output was 'limited to specified cities'. PolicyEngine includes Indiana county tax in local_income_tax at the default Adams County rate of 1.6% on $84,302 of Indiana AGI, giving $1,348.83."
+us,scenario_067,local_income_tax,inkling,prompt_ambiguity,state_local_rule,False,"Applied only the listed city taxes and returned $0, leaving out the Indiana county income tax that PolicyEngine includes. That tax is 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,kimi-k2.6,prompt_ambiguity,state_local_rule,False,"Concluded that Indiana imposes none of the four captured city taxes and stopped there, missing that PolicyEngine's local_income_tax also includes Indiana county income tax. Adams County's 1.6% on $84,302 of Indiana AGI is $1,348.83."
+us,scenario_067,local_income_tax,kimi-k3,prompt_ambiguity,state_local_rule,False,"Checked only NYC, Philadelphia, KC and St. Louis liability and returned $0. PolicyEngine includes Indiana county income tax in this output: 1.6% (Adams County default) × $84,302 Indiana AGI ($87,302 minus three $1,000 exemptions) = $1,348.83."
+us,scenario_067,local_income_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"Recognized the Indiana county income tax but set it to $0 because no county was given. PolicyEngine assigns a default county (Adams, 1.6%) and applies it to Indiana AGI of $84,302, giving $1,348.83."
+us,scenario_067,local_income_tax,muse-spark-1.3,prompt_ambiguity,state_local_rule,False,"Counted only the four named city taxes and returned $0. PolicyEngine's local_income_tax also includes Indiana county income tax: 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"Treated the absence of an NYC, Philadelphia, KC or St. Louis location as meaning zero local tax, missing Indiana county income tax. PolicyEngine computes that tax at the default Adams County rate of 1.6% on $84,302 of Indiana AGI, giving $1,348.83."
+us,scenario_067,local_income_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"Explicitly put Indiana county taxes outside this output, but PolicyEngine includes them in local_income_tax. Adams County's 1.6% on Indiana AGI of $84,302 ($87,302 minus $3,000 of exemptions) is $1,348.83."
+us,scenario_067,local_income_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"Took the four named city taxes as the only modeled local taxes and returned $0. PolicyEngine also models Indiana county income tax in this variable: 1.6% × $84,302 Indiana AGI = $1,348.83."
+us,scenario_067,local_income_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"Said local income taxes apply only in the listed cities and missed the Indiana county income tax that PolicyEngine includes in local_income_tax. That tax is the default Adams County rate of 1.6% on $84,302 of Indiana AGI, or $1,348.83."
us,scenario_067,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model first got the correct $6,300.54 from 7.65% of the head's $82,360. It then made up spouse wages by applying the head's $67 hourly rate to the spouse's 50 weekly hours, even though the prompt says unlisted numbers are 0. After trying several inconsistent versions of that guess, it submitted an unexplained $12,554.41 instead of the head-only figure."
us,scenario_067,payroll_tax,claude-haiku-5.5,llm_error,payroll_tax_base,False,"The model assumed the spouse had wages that the prompt does not list, even though the spouse's wages are $0. At 7.65%, its $8,500 answer implies about $111,000 of taxed wages, roughly $28,700 more than the head's $82,360. Taxing only the head's wages gives $6,300.54."
us,scenario_067,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model used the right base and rates but rounded each piece to whole dollars ($5,106 and $1,194). It then added them to $6,299 instead of $6,300 and blamed a wage limit that does not apply at $82,360. The unrounded total is $5,106.32 + $1,194.22 = $6,300.54."
@@ -6079,25 +6149,26 @@ us,scenario_067,payroll_tax,gemini-3.5-flash-lite,llm_error,payroll_tax_base,Fal
us,scenario_067,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly said payroll tax applies only to the head's $82,360 of wages and that no Additional Medicare Tax is due. Its $5,727.14 is still $573.40 short of the correct $5,106.32 + $1,194.22 = $6,300.54. That figure equals 7.65% of only about $74,864 of wages, which shows an arithmetic error in combining the Social Security and Medicare pieces."
us,scenario_067,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly calculated $5,106 Social Security and $1,194 Medicare on the head's $82,360. It then described the total as a range of $6,300 to $6,400 and arbitrarily submitted $6,376 instead of adding the two pieces to get $6,300.54."
us,scenario_067,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_067,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated Dependent 1's 'is disabled' flag as satisfying SSI's disability criterion and computed (994 − 880) × 12 = $1,368. The engine carries is_ssi_aged_blind_disabled=False for dependent1, so no member is categorically eligible and SSI is $0."
-us,scenario_067,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"The model counted the disabled 23-year-old as an SSI-eligible adult with no parental deeming and paid (994 − 880) × 12 = $1,368. In the engine, dependent1 has is_ssi_aged_blind_disabled=False, which bars SSI entirely and gives $0."
-us,scenario_067,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"The model assumed Dependent 1 qualifies as SSI-disabled and subtracted $10,560 of countable disability income from an $11,928 FBR to get $1,368. The engine marks dependent1 is_ssi_aged_blind_disabled=False, so the categorical test fails and SSI is $0."
-us,scenario_067,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as SSI-disabled and netted $10,560 of countable unearned income against an FBR of $991/month, below the actual 2026 rate of $994, to get $1,334. The engine sets is_ssi_aged_blind_disabled=False for dependent1, so no one is categorically eligible and SSI is $0."
-us,scenario_067,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model computed (991 − 880) × 12 = $1,332 for the disabled dependent, using an FBR $3 below the 2026 rate of $994. More importantly, it treated dependent1 as SSI-disabled even though the engine has is_ssi_aged_blind_disabled=False, which makes SSI $0."
-us,scenario_067,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model treated the disabled adult dependent as SSI-eligible and subtracted $10,560 from an assumed $11,889 annual FBR ($990.75/month instead of $994) to get $1,329. The engine sets dependent1 is_ssi_aged_blind_disabled=False, so SSI is $0."
-us,scenario_067,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"The model treated dependent1 as SSI-disabled, but the engine has is_ssi_aged_blind_disabled=False, which makes SSI $0. It also used the 2025 FBR of $967 instead of the 2026 rate of $994, and took the $20 general exclusion once a year ($10,800 − $20) instead of monthly ($240/year), which produced $824."
-us,scenario_067,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"The model named Dependent 1 the only SSI-eligible member because of the disabled flag and paid $114/month ($994 − $880), or $1,368. In the engine, dependent1 has is_ssi_aged_blind_disabled=False, so nobody passes the categorical test and SSI is $0."
-us,scenario_067,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model applied the $11,928 annual FBR less $10,560 of countable disability income to the disabled dependent and got $1,368. The engine's is_ssi_aged_blind_disabled=False for dependent1 blocks eligibility, giving $0."
-us,scenario_067,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model treated the disabled adult dependent as SSI-eligible with no deeming and computed $11,928 − $10,560 = $1,368. The engine sets is_ssi_aged_blind_disabled=False for dependent1, so SSI is $0."
-us,scenario_067,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model paid the disabled adult $114/month after netting $880 of countable income against the $994 FBR, for $1,368. The engine marks dependent1 is_ssi_aged_blind_disabled=False, so no categorical pathway exists and SSI is $0."
-us,scenario_067,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model reduced the $994 FBR by $880 of countable disability income and annualized $114/month to $1,368 for Dependent 1. The engine carries is_ssi_aged_blind_disabled=False for dependent1, so SSI is $0."
-us,scenario_067,ssi,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"The model treated the disabled adult as SSI-eligible and subtracted disability income after the general exclusion from the 2026 individual maximum, getting $1,368. The engine has is_ssi_aged_blind_disabled=False for dependent1, so the categorical test fails and SSI is $0."
-us,scenario_067,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"The model declared Dependent 1 categorically eligible based on the disabled flag and computed ($994 − $880) × 12 = $1,368. The engine sets is_ssi_aged_blind_disabled=False for dependent1, so no member is categorically eligible and SSI is $0."
-us,scenario_067,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model treated dependent1 as SSI-disabled even though the engine has is_ssi_aged_blind_disabled=False, which makes SSI $0. It also took the $20 general exclusion once a year ($10,800 − $20 = $10,780) instead of $20 a month ($240), and used an $11,892 FBR instead of $11,928, which produced $1,112."
-us,scenario_067,ssi,inkling,prompt_ambiguity,age_disability,False,"The model treated Dependent 1's disability as meeting SSI's categorical criterion and paid ($994 − $880) × 12 = $1,368. The engine sets is_ssi_aged_blind_disabled=False for dependent1, so SSI is $0."
-us,scenario_067,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"The model applied adult no-deeming rules to the disabled 23-year-old and computed $114/month, or $1,368/year. In the engine, dependent1's is_ssi_aged_blind_disabled is False, so there is no SSI eligibility and the benefit is $0."
-us,scenario_067,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"The model used a made-up FBR of $1,913/month (not the $994 individual rate for 2026), skipped the $20 general income exclusion, and got $22,956 − $10,800 = $12,156. On top of that, it treated dependent1 as SSI-disabled even though the engine has is_ssi_aged_blind_disabled=False, which makes SSI $0."
-us,scenario_067,ssi,muse-wonder,prompt_ambiguity,age_disability,False,"The model treated the disabled dependent as SSI-eligible and paid $114/month ($994 − $880), or $1,368/year. The engine carries is_ssi_aged_blind_disabled=False for dependent1, so no member is categorically eligible and SSI is $0."
+us,scenario_067,ssi,claude-fable-5.1,prompt_ambiguity,age_disability,False,"The model treated the 23-year-old as SSI-eligible through disability and computed ($994 FBR − $880 countable unearned income) × 12 = $1,368. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, and Head and Spouse are under 65, so no member passes the categorical test and SSI is $0."
+us,scenario_067,ssi,claude-fable-5.2,prompt_ambiguity,age_disability,False,"The model correctly excluded Head and Spouse as under 65 and not disabled, but treated Dependent 1 as SSI-disabled and computed $114/month × 12 = $1,368. The engine's is_ssi_aged_blind_disabled is False for Dependent 1, so Dependent 1 is not categorically eligible and SSI is $0."
+us,scenario_067,ssi,claude-opus-5.5,prompt_ambiguity,age_disability,False,"The model gave the disabled adult dependent SSI of ($994 − $880) × 12 = $1,368 after noting there is no parental deeming. The engine does not count Dependent 1 as SSI aged, blind, or disabled (is_ssi_aged_blind_disabled=False), so the categorical test fails and SSI is $0."
+us,scenario_067,ssi,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"The model computed $11,928 − $10,560 countable disability income = $1,368, assuming Dependent 1 meets SSI disability. The engine sets is_ssi_aged_blind_disabled=False for every member, including Dependent 1, so SSI is $0 before any income test."
+us,scenario_067,ssi,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as SSI-disabled and computed a benefit, using an underestimated 2026 FBR of about $991/month instead of $994. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, and no other member is aged or disabled, so SSI is $0."
+us,scenario_067,ssi,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"The model gave Dependent 1 SSI of ($991 − $880) × 12 = $1,332, treating the dependent as categorically disabled and also understating the 2026 FBR ($994). In the engine, Dependent 1 fails is_ssi_aged_blind_disabled, as do Head (60) and Spouse (58), so SSI is $0."
+us,scenario_067,ssi,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"The model treated the adult disabled child as SSI-eligible and subtracted $10,560 countable income from an assumed $11,889 annual maximum ($990.75/month instead of $994). The engine marks Dependent 1 as not aged, blind, or disabled for SSI, so no member is categorically eligible and SSI is $0."
+us,scenario_067,ssi,glm-5.2,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as SSI-disabled and made two further errors: it used the 2025 FBR of $967 instead of $994, and applied the $20 general exclusion once a year instead of monthly ($10,800 − $20). The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, and Head and Spouse are under 65, so SSI is $0."
+us,scenario_067,ssi,gpt-5.5,prompt_ambiguity,age_disability,False,"The model said only the disabled 23-year-old is SSI-eligible and computed ($994 − $880) × 12 = $1,368. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1 too, so no member passes the aged/blind/disabled test and SSI is $0."
+us,scenario_067,ssi,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"The model subtracted $10,560 countable disability income from an $11,928 annual FBR, treating Dependent 1 as SSI-disabled. The engine's is_ssi_aged_blind_disabled is False for Dependent 1, so the categorical test fails and SSI is $0."
+us,scenario_067,ssi,gpt-6-astra,prompt_ambiguity,age_disability,False,"The model treated the disabled adult dependent as categorically eligible and computed $11,928 − $10,560 = $1,368. In the engine Dependent 1 is not SSI aged, blind, or disabled (is_ssi_aged_blind_disabled=False), so SSI is $0."
+us,scenario_067,ssi,gpt-6-sol,prompt_ambiguity,age_disability,False,"The model gave the disabled adult $114/month ($994 FBR − $880 countable income), or $1,368 a year, assuming SSI disability eligibility. The engine sets is_ssi_aged_blind_disabled=False for all three members, so SSI is $0."
+us,scenario_067,ssi,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"The model reduced the $994 monthly FBR by $880 countable disability income for Dependent 1, treating the dependent as SSI-disabled. The engine flags Dependent 1 with is_ssi_aged_blind_disabled=False, and Head and Spouse are under 65, so SSI is $0."
+us,scenario_067,ssi,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"The model applied the 2026 individual maximum minus countable disability income after the $20 exclusion ($1,368), assuming Dependent 1 qualifies as disabled. In the engine Dependent 1 does not meet the SSI aged/blind/disabled criterion, so SSI is $0."
+us,scenario_067,ssi,grok-4.7,prompt_ambiguity,age_disability,False,"The model called Dependent 1 the only categorically eligible member and computed $114/month × 12 = $1,368. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1 as well, so no member is categorically eligible and SSI is $0."
+us,scenario_067,ssi,grok-build-0.1,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as SSI-disabled, used an extrapolated FBR of $11,892 instead of $11,928, and applied the $20 general exclusion once a year rather than $240 annually. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, and Head and Spouse are under 65, so SSI is $0."
+us,scenario_067,ssi,inkling,prompt_ambiguity,age_disability,False,"The model computed ($994 − $880) × 12 = $1,368 for Dependent 1 on the premise that the dependent is SSI-disabled. The engine returns is_ssi_aged_blind_disabled=False for Dependent 1, so SSI is $0."
+us,scenario_067,ssi,kimi-k3,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as a disabled adult eligible for SSI without parental deeming and computed $114/month × 12 = $1,368. The engine does not count Dependent 1 as SSI aged, blind, or disabled, so the categorical test fails and SSI is $0."
+us,scenario_067,ssi,minimax-m3,prompt_ambiguity,age_disability,False,"The model treated Dependent 1 as SSI-disabled, used a $1,913/month FBR that matches no SSI rate (2026 individual FBR is $994), and skipped the $20 general income exclusion. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1, and Head and Spouse are under 65, so SSI is $0."
+us,scenario_067,ssi,muse-wonder,prompt_ambiguity,age_disability,False,"The model subtracted $880 countable disability income from the $994 FBR for Dependent 1, giving $1,368 a year and assuming SSI disability eligibility. The engine sets is_ssi_aged_blind_disabled=False for Dependent 1 and for both adults, so SSI is $0."
us,scenario_067,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Added a made-up $500 extra exemption for the disabled dependent, so its exemptions came to $3,500 instead of $3,000. It also used a 2.9% rate instead of 2.95%. It then cut its own $2,430 figure down to $2,295 with an unexplained 'minor deductions' adjustment. The correct result is $84,302 × 2.95% = $2,486.91."
us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Started from federal taxable income after a $28,700 federal standard deduction instead of from federal AGI less Indiana's $1,000-per-person exemptions. It used a 3.15% rate instead of 2.95% and invented health-premium and medical-expense credits. Its final $4,168 does not follow from any of its own steps."
us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"Used a 3.05% rate instead of the 2026 rate of 2.95%. Its $4,500 answer implies about $147,500 of taxable income, far above the correct $84,302 ($87,302 AGI less $3,000 of exemptions)."
@@ -6290,44 +6361,45 @@ us,scenario_070,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"T
us,scenario_070,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model conflated regular Medicare tax with the Additional Medicare Tax and set all Medicare tax to zero because wages were below the additional-tax threshold. The ordinary 1.45% Medicare tax applies from the first dollar of covered wages, adding $551 to the correctly computed $2,356 Social Security tax."
us,scenario_070,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages from $38,000 to $37,700 by assigning unlisted pre-tax treatment to the $300 health premium. Applying 6.2% Social Security and 1.45% Medicare rates to the full wages produces $2,907."
us,scenario_070,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly derived $2,356 of Social Security tax plus $551 of Medicare tax, then replaced the exact $2,907 total with $2,970 under an invalid claim of rounding. Rounding $2,907 does not produce $2,970, and the requested annual amount is the exact derived total."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The AGI of $36,270 was correct, but the model subtracted the 2025 Illinois personal exemption of $2,850 instead of the 2026 indexed amount of $2,925. That leaves taxable income $75 too high: 33,420 × 4.95% = $1,654.29, not $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model left the $1,485 pre-tax traditional 401(k) contribution in the base ($37,755 instead of about $36,270). It then treated a $2,425 exemption as a $120 tax credit instead of subtracting $2,925 from income. Even its own numbers don't add up: $1,869 − $120 is not $1,154."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"The model used an Illinois base of about $20,000 that has no support in the facts. Illinois has no retirement-contribution subtraction beyond what federal AGI already excludes. The correct base is federal AGI of $36,270 minus the $2,925 exemption, which is $33,345 and gives $1,650.56, not $1,000."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly derived AGI of $36,270 but used the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. It computed $1,654.29 and then reported $1,659.83 with no supporting calculation, moving further from the correct $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly derived AGI of $36,270 but cycled through old exemption amounts ($2,775, then $2,425) instead of the 2026 amount of $2,925. It then dropped both of those results and submitted $1,726 based on an unexplained taxable income of $34,870."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used the 2025 exemption of $2,850 instead of the 2026 amount of $2,925, which gave about $1,655. It then raised this to $1,699 by 'rounding' with no basis, instead of computing (36,270 − 2,925) × 4.95% = $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model wrongly added the IRA and student loan interest deductions back to the Illinois base; Illinois starts from federal AGI with no addback for either. It also subtracted the $300 health premium as pre-tax wages, which the reference AGI does not do. On top of that it used the outdated $2,425 exemption instead of $2,925."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model claimed Illinois disallows the $276 student loan interest deduction, but Illinois base income starts from federal AGI, which already includes that deduction. It also used the $2,850 exemption instead of $2,925, then raised its own $1,668 result to $1,690 with a made-up 'exemption phase-in' adjustment."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"The AGI of $36,270 was correct, but the model applied the 2025 exemption of $2,850 instead of the 2026 indexed exemption of $2,925. The result was about $1,654 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"The model invented an Illinois standard deduction of $2,450 and stacked it on a $2,450 personal exemption. Illinois has no standard deduction, only one indexed exemption of $2,925 for 2026, so it subtracted $4,900 instead of $2,925."
-us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an AGI of $35,670, which is $600 too low, consistent with wrongly treating both $300 health premiums as excluded from income. It also used a $2,842 exemption instead of the 2026 amount of $2,925."
-us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model applied the 2025 exemption of $2,850 to the correct $36,270 AGI instead of the 2026 indexed exemption of $2,925. The result, $1,654.29, is $3.73 too high."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model used the 2024 exemption of $2,775 instead of $2,925. It then subtracted a $135 'renter's credit' that Illinois does not offer: the Illinois property tax credit requires property tax actually paid on a principal residence, and rent does not qualify."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model cited the outdated $2,775 exemption instead of $2,925 for 2026. Its $1,716 answer implies taxable income of about $34,667, which matches neither $36,270 − $2,775 nor the correct $33,345."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model subtracted a rough $2,500 exemption from the $36,270 AGI instead of the 2026 indexed exemption of $2,925. That overstates taxable income by about $425 and gives $1,671 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's AGI of $35,970 wrongly excludes the $300 health premium as a pre-tax payroll reduction; the reference AGI is about $36,270. It also subtracted the 2024 exemption of $2,775 instead of the 2026 amount of $2,925."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no derivation. Its $1,683 answer implies Illinois taxable income of about $34,000, which matches applying an exemption of only about $2,270 to the $36,270 AGI instead of the 2026 exemption of $2,925 (taxable $33,345)."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied the 2025 exemption of $2,850 to the correct $36,270 AGI instead of the 2026 indexed exemption of $2,925. The result was $1,654.29 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used the 2024 Illinois exemption of $2,775 instead of the 2026 amount of $2,925. That leaves taxable income $150 too high and gives $1,658 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"The model subtracted the 2024 exemption of $2,775 instead of the 2026 indexed exemption of $2,925 from the $36,270 AGI. The result was $1,658 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model left the $276 student loan interest deduction out of federal AGI ($36,546 instead of $36,270) and used the outdated $2,425 exemption instead of $2,925. It computed $1,689 and then submitted an inconsistent $1,626."
-us,scenario_070,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"The AGI of $36,270 was correct, but the model used the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. The result was about $1,654 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model claimed the exemption and retirement deductions reduce the tax to zero. The only Illinois exemption is $2,925, so about $33,345 remains taxable at 4.95%, and the tax is $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed the retirement and student loan adjustments eliminate the Illinois tax. Those adjustments reduce AGI by only about $1,830, to $36,270, and after the $2,925 exemption the flat 4.95% rate still produces $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model left the $276 student loan interest deduction out of federal AGI ($36,546 instead of $36,270). It also used a made-up $1,824 exemption instead of the 2026 Illinois exemption of $2,925."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used the 2025 exemption of $2,850 instead of the 2026 indexed exemption of $2,925 on the correct $36,270 AGI. The result was $1,654.29 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave no derivation. Its $1,777 answer implies Illinois taxable income of about $35,900, which matches subtracting almost no exemption from the base. The correct calculation subtracts the full $2,925 exemption from $36,270 AGI, leaving $33,345."
-us,scenario_070,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model subtracted the 2024 exemption of $2,775 instead of the 2026 indexed amount of $2,925. The result was $1,658 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model's AGI of $35,970 wrongly excludes the $300 health premium from income; the reference AGI is about $36,270. It also used the outdated $2,425 exemption instead of $2,925, and the two errors partly cancel to give $1,660."
-us,scenario_070,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used the 2024 exemption of about $2,775 instead of the 2026 indexed exemption of $2,925. That overstates taxable income by $150 and gives $1,658."
-us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model claimed Illinois has no personal exemption and taxed the full $36,270 AGI. Illinois allows a $2,925 personal exemption for 2026, which cuts taxable income to $33,344.67 and the tax to $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The base income of $36,270 was correct, but the model subtracted the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. The result was $1,658 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model applied a federal-style standard deduction to get taxable income of about $20,696, but Illinois has no standard deduction. Illinois taxes federal AGI of $36,270 minus only the $2,925 exemption."
-us,scenario_070,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"The model used the 2025 exemption of $2,850 instead of the 2026 indexed exemption of $2,925 on the correct $36,270 base. The result was about $1,654 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model subtracted an outdated $2,450 exemption from the $36,270 AGI instead of the 2026 amount of $2,925. That overstates taxable income by $475 and gives $1,674.09."
-us,scenario_070,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model estimated the 2026 exemption at about $2,900 instead of the actual indexed $2,925. That leaves taxable income $25 too high: $33,370 × 4.95% = $1,651.82, not $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model subtracted the exemption twice: once from income, then again as a $2,800 nonrefundable credit against the tax itself, which zeroed it out. The Illinois exemption only reduces taxable income. The model also left out the student loan interest deduction from AGI."
-us,scenario_070,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model described the right method (AGI minus exemption, times 4.95%). But its $1,658.66 answer implies taxable income of about $33,508, which matches subtracting an exemption of about $2,762 instead of the 2026 indexed amount of $2,925."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its federal AGI of $36,270 was correct, but it subtracted the 2025 Illinois personal exemption of $2,850 instead of the 2026 indexed amount of $2,925. That overstated taxable income by $75, and the tax by $3.73."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,thresholds_rates,False,"It used an assumed $2,850 exemption (the 2025 value) instead of the 2026 Illinois exemption of $2,925. That gave taxable income of $33,420 instead of $33,344.67 and tax of $1,654.29 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It never excluded the $1,485 traditional 401(k) from wages, so its base was $37,755 instead of $36,269.67. It then treated the outdated $2,425 exemption as a $120 credit when it is a subtraction from base income worth $2,925. Its final $1,154 does not follow from its own figures ($1,869 − $120 = $1,749)."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It invented an Illinois 'retirement subtraction' and cut the base to about $20,000. The correct base is federal AGI of $36,269.67 minus only the $2,925 exemption, which leaves $33,344.67 taxed at 4.95%."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It computed $1,654.29 using the 2025 exemption of $2,850 instead of the 2026 amount of $2,925. It then reported $1,659.83 with no supporting computation, which moved it further from the correct $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the 2024 exemption of $2,775 instead of $2,925 and got $1,658. It then switched without explanation to a taxable income of $34,870 and reported $1,726. The correct figure is 4.95% × $33,344.67."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It started from the 2025 exemption of $2,850, not the 2026 $2,925, and got about $1,655. It then raised the figure to $1,699 for unexplained 'rounding' instead of computing 4.95% × ($36,269.67 − $2,925)."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It wrongly added back the IRA deduction and student loan interest, but Illinois base income starts from federal AGI with no such add-backs. It also treated the $300 health premium as a pre-tax wage exclusion and used the outdated $2,425 exemption instead of $2,925, ending at $33,890 taxable instead of $33,344.67."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It left the $276 above-the-line student loan interest deduction out of federal AGI ($36,546 instead of $36,269.67), and Illinois base income inherits that deduction. It also used the $2,850 exemption instead of $2,925 and then inflated its $1,668 result to $1,690 for no stated reason."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It applied the 2025 personal exemption of $2,850 instead of the 2026 indexed $2,925. That left taxable income $75 too high and gave $1,654 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted a $2,450 'standard deduction' that Illinois does not have, on top of a $2,450 exemption. Illinois allows only the personal exemption, which is $2,925 for 2026, so taxable income is $33,344.67, not $31,370."
+us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It understated federal AGI at $35,670, $600 below $36,269.67, consistent with wrongly subtracting both $300 health premiums. It also used a $2,842 exemption instead of the 2026 amount of $2,925."
+us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"Its federal AGI was right, but it used the 2025 Illinois exemption of $2,850 instead of the 2026 $2,925. That gave $1,654.29 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It invented a $135 nonrefundable renter's credit, but the Illinois property tax credit applies only to property tax paid on a principal residence, so renters get nothing. It also used the 2024 exemption of $2,775 instead of the 2026 $2,925."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It cited the outdated $2,775 exemption instead of $2,925. Its $1,716 implies about $34,667 of taxable income, roughly $1,320 above the correct $36,269.67 − $2,925 = $33,344.67, so it also overstated the base."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It subtracted a rough $2,500 exemption instead of the 2026 Illinois exemption of $2,925. That overstated taxable income by about $425 and gave $1,671."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It reduced federal AGI to $35,970 by treating the $300 health premium as a wage exclusion; the correct AGI is $36,269.67. It also used the 2024 exemption of $2,775 instead of $2,925."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It showed no base calculation. Its $1,683 implies $34,000 of taxable income, which means an exemption of only about $2,270 instead of the 2026 $2,925 subtracted from AGI of $36,269.67."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used the 2025 Illinois personal exemption of $2,850 instead of the 2026 indexed $2,925. The result was $1,654.29 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It subtracted the 2024 Illinois exemption of $2,775 instead of the 2026 amount of $2,925. That overstated taxable income by $150 and gave $1,658."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the outdated $2,775 exemption, the 2024 value, instead of $2,925 for 2026, and got $1,658 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left the $276 student loan interest deduction out of federal AGI ($36,546) and used the outdated $2,425 exemption instead of $2,925, which gave $1,689. It then reported an unrelated $1,626 that its own work does not support."
+us,scenario_070,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It applied the 2025 exemption of about $2,850 instead of the 2026 $2,925. That gave taxable income of $33,420 and tax of $1,654 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the exemption and retirement deductions wipe out the Illinois liability. In fact, after the $2,925 exemption, $33,344.67 of base income remains taxable at the flat 4.95% rate."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It reported zero Illinois tax, but the retirement and student-loan adjustments only bring AGI down to $36,269.67. After the $2,925 exemption, $33,344.67 is still taxed at 4.95%."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a made-up $1,824 exemption instead of the 2026 Illinois exemption of $2,925. It also left the $276 student loan interest deduction out of AGI ($36,546 instead of $36,269.67)."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It subtracted the 2025 exemption of $2,850 instead of the 2026 indexed $2,925, which gave $1,654.29 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave no computation. Its $1,777 implies about $35,900 of taxable income, which means it applied almost none of the $2,925 exemption to the $36,269.67 AGI."
+us,scenario_070,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the 2024 exemption of $2,775 instead of the 2026 amount of $2,925. That overstated taxable income by $150 and gave $1,658."
+us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It reduced AGI to $35,970 by subtracting the $300 health premium, which does not reduce AGI here. It also used the outdated $2,425 exemption instead of $2,925, so the two errors partly offset and it landed at $1,660."
+us,scenario_070,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It subtracted the 2024 exemption of about $2,775 instead of the 2026 indexed $2,925, giving $1,658 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It wrongly asserted that Illinois has no personal exemption and taxed the full $36,270. Illinois subtracts a $2,925 exemption in 2026, which leaves $33,344.67 taxable."
+us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Its base income was correct, but it applied the 2024 exemption of $2,775 instead of the 2026 $2,925, overstating the tax by about $7.42."
+us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It applied a federal-style standard deduction and got taxable income of about $20,696, but Illinois has no standard deduction. The base is AGI of $36,269.67 minus only the $2,925 exemption. It then adjusted to $1,106 with no basis."
+us,scenario_070,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used the 2025 exemption of $2,850 instead of the 2026 $2,925, which gave $1,654 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It subtracted a $2,450 exemption instead of the 2026 Illinois exemption of $2,925. That overstated taxable income by $475 and gave $1,674.09."
+us,scenario_070,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It estimated the 2026 exemption at $2,900 instead of the actual indexed $2,925. Taxable income came out $25 too high and the result was $1,651.82 instead of $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted the Illinois exemption twice: once as a deduction and again as a $2,800 nonrefundable credit that zeroed out the tax. The exemption is only a $2,925 subtraction from base income, and there is no exemption credit. It also left the student loan interest deduction out of AGI."
+us,scenario_070,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"Its $1,658.66 implies taxable income of $33,508 and an exemption of about $2,760, consistent with the 2024 $2,775 figure. The 2026 indexed exemption is $2,925."
us,scenario_070,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly found that the childless federal EITC was fully phased out but then invented a $102 Illinois EITC despite Illinois calculating its credit as a percentage of the federal amount. The age expansion changes categorical eligibility only; it does not override the federal income phaseout, so 20% of a zero federal EITC yields $0."
us,scenario_071,head_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model declared $36,542 of income below New York’s aged Medicaid limits and simultaneously treated $62,291 as below a resource limit it described as $15,900, reversing both comparisons. It therefore invented an aged Medicaid pathway even though the person’s engine category is NONE."
us,scenario_071,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model invoked unspecified senior disregards to reduce $36,542 below an aged Medicaid limit of roughly $20,000 without calculating any applicable disregard. No aged pathway is established, and the engine assigns medicaid_category = NONE."
@@ -6763,113 +6835,115 @@ us,scenario_077,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,Fals
us,scenario_077,snap,gemini-3.7-flash,llm_error,other,False,"The model’s explanation correctly states that the one-person household exceeds the SNAP net-income limit and explicitly concludes “value = 0,” but it submitted 276. The numeric output contradicts its own eligibility conclusion and should have been 0."
us,scenario_077,snap,grok-4.3,llm_error,thresholds_rates,False,"The model labeled the household low-income without applying the one-person SNAP net-income and benefit calculations to its $26,006 of wages. After the allowable earned-income and standard deductions, the household does not generate an eligible allotment, so the asserted $2,400 benefit has no valid computation."
us,scenario_077,snap,grok-4.6,llm_error,categorical_eligibility,False,The model incorrectly awarded the $23 minimum allotment after determining that the regular formula produced zero. Broad-based categorical eligibility does not entitle a household failing the applicable net-income or benefit test to the minimum allotment; the minimum applies only to an otherwise eligible one- or two-person household.
-us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 deduction to get $315.18, then raised the answer to $496 with no basis. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_later_law,taxable_income_or_deductions,False,"The model used the 2025 base standard deduction of $12,500. That produced $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_later_law,taxable_income_or_deductions,False,"The model said Louisiana has no individual income tax. Louisiana taxes income at a flat 3%. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_later_law,taxable_income_or_deductions,False,"The model assumed a large personal exemption plus the standard deduction wiped out the tax base. The 2025 reform repealed the personal exemption, and for this household the only subtraction is the standard deduction. The frozen reference uses a $12,835 deduction, leaves $10,171.18 taxable at 3% and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_later_law,taxable_income_or_deductions,False,"The model applied the flat 3% rate correctly but used the unindexed $12,500 standard deduction. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 as the single deduction. The model submitted $315. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,reference_later_law,taxable_income_or_deductions,False,"The model reached about $315 using the unindexed $12,500 deduction, then raised it to $446 by citing capital-loss treatment and rounding. The $3,000 capital-loss cap is already reflected in its $23,006 AGI. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 standard deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_later_law,taxable_income_or_deductions,False,"The model subtracted a $4,500 personal exemption and an $801 federal-income-tax deduction from $23,006 AGI, leaving $17,705 taxable at 3% and about $531 of tax. Louisiana ended the federal income tax deduction after 2021; the 2025 reform replaced the personal exemption with a standard deduction. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_later_law,taxable_income_or_deductions,False,"The model computed about $315 with the unindexed $12,500 deduction, then added unexplained 'federal-based adjustments' to reach $448. For this household the only subtraction from AGI is the standard deduction. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_later_law,taxable_income_or_deductions,False,"The model assumed a $12,500 standard deduction. That gives $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_later_law,taxable_income_or_deductions,False,"The model's $7,606 taxable base implies a $15,400 deduction, which is not Louisiana's amount. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 standard deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_later_law,taxable_income_or_deductions,False,"The model used a federal-income-tax deduction that Louisiana ended after 2021, a $4,500 deduction plus a $1,000 exemption, and graduated 1.85%/3.5% brackets. Louisiana has applied a flat 3% rate since 2025. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $8,389 of ESI premiums from the gross wage input, which put AGI at $14,617 instead of $23,006. It then used the repealed $4,500 exemption and the old 1.85% bracket. The frozen reference applies the flat 3% rate after a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_later_law,taxable_income_or_deductions,False,"The model gave no derivation, and $158 is below the frozen reference. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994. The low figure matches taking oversized deductions or using the old 1.85% bottom bracket instead of the flat 3% rate."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_later_law,taxable_income_or_deductions,False,"The model subtracted ESI premiums from gross wages (AGI $14,617 instead of $23,006) and then took two $4,500 subtractions. It taxed the remainder at the repealed 1.85% bracket. The frozen reference applies a flat 3% after a single $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_later_law,taxable_income_or_deductions,False,"The model reduced AGI to $14,617 by subtracting ESI premiums from the gross wage input. It then used the repealed $4,500 exemption and 1.85% bracket. The frozen reference applies the flat 3% rate to AGI minus a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_later_law,taxable_income_or_deductions,False,"The model said deductions eliminate the tax. For this household the only subtraction is the standard deduction. The frozen reference uses a $12,835 deduction, leaves $10,171.18 of the $23,006.18 AGI taxable at 3% and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_later_law,taxable_income_or_deductions,False,"The model's $315.18 implies the unindexed $12,500 deduction. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_later_law,taxable_income_or_deductions,False,"The model cited a 'standard deduction and personal exemption' and gave $130 in its reasoning but submitted $152, so its own figures disagree. Louisiana has no personal exemption after the 2025 reform. The frozen reference applies a flat 3% to AGI minus a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_later_law,taxable_income_or_deductions,False,"The model wrongly said PolicyEngine does not model Louisiana income tax. The frozen reference applies Louisiana's flat 3% to AGI minus a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-4-argon,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 base standard deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,reference_later_law,taxable_income_or_deductions,False,"The model correctly capped the capital loss and applied the flat 3% rate. It used the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,glm-5.3,reference_later_law,taxable_income_or_deductions,False,"The model taxed income at the repealed 1.85% bottom bracket rate instead of Louisiana's flat 3%. It also used the $12,500 deduction. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_later_law,taxable_income_or_deductions,False,"The model claimed taxable income falls below a lowest-bracket threshold. Louisiana has a flat 3% rate with no zero bracket. The frozen reference uses a $12,835 deduction, leaves $10,171.18 taxable and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_later_law,taxable_income_or_deductions,False,"The model assumed zero tax because wage income is low. It never applied Louisiana's standard deduction and flat 3% rate to its AGI. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 single standard deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_later_law,taxable_income_or_deductions,False,"The model applied the $12,500 deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_later_law,taxable_income_or_deductions,False,"The model stacked a $4,500 personal exemption on top of the standard deduction, but the 2025 reform repealed that exemption. It also used the $12,500 deduction. The frozen reference uses a $12,835 deduction, leaves $10,171.18 taxable at 3% and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-astra,reference_later_law,taxable_income_or_deductions,False,"The model subtracted a $12,875 standard deduction, which it called inflation-adjusted, from a $23,006 AGI and taxed the remaining $10,131 at 3%, for $303.93. That $12,875 figure is Louisiana's withholding and estimated-tax amount. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-luna,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 standard deduction. That gives $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-sol,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the $12,500 base deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 single deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-6.1-terra,reference_later_law,taxable_income_or_deductions,False,"The model applied the unindexed $12,500 standard deduction. The model submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,reference_later_law,taxable_income_or_deductions,False,"The model claimed deductions and exemptions reduce taxable income to zero. For this household the only subtraction is the standard deduction. The frozen reference uses a $12,835 deduction, leaves $10,171.18 taxable at 3% and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,reference_later_law,taxable_income_or_deductions,False,"The model used the repealed $4,500 exemption and the old 1.85%/3.5% graduated brackets. Louisiana's 2025 reform replaced these with a flat 3% rate and a standard deduction. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-4.6,reference_later_law,taxable_income_or_deductions,False,"The model applied the flat 3% rate to AGI minus an unindexed $12,500 deduction. That gives $315. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-4.7,reference_later_law,taxable_income_or_deductions,False,"The model correctly noted the repeal of the federal tax deduction but used the $12,500 base standard deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,reference_later_law,taxable_income_or_deductions,False,"The model subtracted ESI premiums from gross wages (AGI $14,617 instead of $23,006) and used the old $4,500 deduction with the repealed 1.85% bracket. The frozen reference applies a flat 3% to AGI minus a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,inkling,reference_later_law,taxable_income_or_deductions,False,"The model used the unindexed $12,500 standard deduction and submitted $315. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for state_income_tax_before_refundable_credits, so there is no answer to score against $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,reference_later_law,taxable_income_or_deductions,False,"The model subtracted $12,500 and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,reference_later_law,taxable_income_or_deductions,False,"The model said Louisiana has no state income tax. Louisiana levies a flat 3% income tax. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,muse-spark-1.3,reference_later_law,taxable_income_or_deductions,False,"The model used the $12,500 standard deduction and submitted $315.18. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,muse-wonder,reference_later_law,taxable_income_or_deductions,False,"The model applied the unindexed $12,500 single deduction. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,ox-alpha,reference_later_law,taxable_income_or_deductions,False,"The model subtracted the federal $16,100 standard deduction plus a repealed $4,500 personal exemption. The frozen Louisiana computation applies its own standard deduction and no separate exemption. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_later_law,taxable_income_or_deductions,False,"The model did not apply the $3,000 capital-loss limit, which put AGI at $18,038 instead of $23,006. It then used the repealed $4,500 exemption and the old 1.85%/3.5% graduated brackets. Louisiana has applied a flat 3% rate since 2025. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
-us,scenario_077,state_income_tax_before_refundable_credits,qwen3.8-max,reference_later_law,taxable_income_or_deductions,False,"The model's $347.24 implies about $11,575 taxable at 3%, which means it subtracted only about $11,431 from AGI. The frozen reference uses a $12,835 deduction and gives $305.14; this output is excluded from scoring under d994."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,reference_later_law,taxable_income_or_deductions,False,"It reached 3% × ($23,006 − $12,500) = $315.18 but then raised the answer to $496 by citing an unspecified 'deduction structure', which has no rule behind it. It also used the un-indexed $12,500 deduction instead of the 2026 indexed $12,835, which gives $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5.1,reference_later_law,taxable_income_or_deductions,False,"It used the 2025 single standard deduction of $12,500 instead of the 2026 inflation-indexed $12,835. That overstated taxable income by $335 and the tax by about $10 ($315.18 vs $305.14)."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5.2,reference_later_law,taxable_income_or_deductions,False,"It correctly applied inflation indexing to the standard deduction but guessed $12,875 instead of the actual 2026 figure of $12,835. That understated taxable income by $40 and gave $303.93 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,reference_later_law,taxable_income_or_deductions,False,"It said Louisiana has no state income tax. Louisiana taxes individual income at a flat 3%, which here applies to $10,171.18 of taxable income (AGI $23,006.18 minus the $12,835 standard deduction), giving $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-5.5,reference_later_law,taxable_income_or_deductions,False,"It claimed the deduction and exemption reduce the taxable base to zero. Louisiana's only offset is the $12,835 standard deduction, which leaves $10,171.18 taxable at 3%."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,reference_later_law,taxable_income_or_deductions,False,"It subtracted the 2025 $12,500 standard deduction instead of the 2026 inflation-indexed $12,835, which gave $315 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,reference_later_law,taxable_income_or_deductions,False,"It treated $12,500 as the 2026 single exemption/deduction, but the deduction is indexed to $12,835 for 2026. Its tax of $315 should be $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,reference_later_law,taxable_income_or_deductions,False,"It computed about $315 using the un-indexed $12,500 deduction, then raised the answer to $446 citing 'capital-loss treatment and rounding', for which no rule exists. The correct result is 3% × ($23,006.18 − $12,835) = $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5.5,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction instead of the 2026 inflation-indexed $12,835. That overstated taxable income by $335 and gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_later_law,taxable_income_or_deductions,False,"It applied the pre-reform $4,500 personal exemption and a deduction for federal income tax paid. The 2025 reform repealed both and replaced them with a single standard deduction of $12,835 for 2026, so it got $531 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,reference_later_law,taxable_income_or_deductions,False,"It reached about $315 using the $12,500 deduction, then raised it to $448 for 'federal-based deductions' and 'personal exemption credits', which do not exist under the 2026 flat-tax system. The correct tax uses the indexed $12,835 deduction and is $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_later_law,taxable_income_or_deductions,False,"It assumed the un-indexed $12,500 standard deduction, but the 2026 indexed amount is $12,835. That gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_later_law,taxable_income_or_deductions,False,"Its taxable base of $7,606 implies a $15,400 deduction, which is a federal-style amount rather than Louisiana's $12,835 single standard deduction. The correct base is $10,171.18, giving $305.14 instead of $228.18."
+us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,reference_later_law,taxable_income_or_deductions,False,"It used the 2025 $12,500 standard deduction instead of the 2026 inflation-indexed $12,835, which gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_later_law,taxable_income_or_deductions,False,"It used repealed pre-2025 rules: a deduction for federal income tax, a $4,500 deduction plus a $1,000 exemption, and graduated 1.85%/3.5% brackets. It should have applied the flat 3% rate to AGI less the $12,835 standard deduction, which gives $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_later_law,taxable_income_or_deductions,False,"It subtracted $12,500 instead of the 2026 inflation-indexed single standard deduction of $12,835, which gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_later_law,taxable_income_or_deductions,False,"It subtracted the $8,389 of employer insurance premiums from the stated gross wages, which put AGI at $14,617 instead of $23,006.18. It then applied the repealed $4,500 exemption and the old 1.85% bottom bracket instead of the $12,835 deduction and the flat 3% rate."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_later_law,taxable_income_or_deductions,False,"It gave no computation. Its $158 does not match the correct derivation, which applies the flat 3% rate to AGI of $23,006.18 less the $12,835 standard deduction and yields $305.14. The number implies a far larger deduction or a lower rate than Louisiana's 2026 rules allow."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_later_law,taxable_income_or_deductions,False,"It cut AGI to $14,617 by subtracting the employer insurance premiums from gross wages. It then subtracted the repealed $4,500 deduction and $4,500 exemption and taxed the rest at the old 1.85% bracket. The correct calculation is 3% of ($23,006.18 − $12,835) = $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,reference_later_law,taxable_income_or_deductions,False,"It reduced AGI to $14,617 by deducting the employer insurance premiums from gross wages and used the pre-reform $4,500 exemption with the 1.85% bracket. Under 2026 law it should have taxed AGI of $23,006.18 less $12,835 at a flat 3%."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_later_law,taxable_income_or_deductions,False,"It claimed deductions and exemptions eliminate the tax. The only offset is the $12,835 standard deduction, which leaves $10,171.18 taxable at 3%, or $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,reference_later_law,taxable_income_or_deductions,False,"Its $315.18 implies it used the un-indexed $12,500 deduction. The 2026 single standard deduction is $12,835, which gives $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.7-flash,reference_later_law,taxable_income_or_deductions,False,"It cited a pre-reform 'standard deduction and personal exemption' structure, and its reasoning ($130) does not even match its submitted answer ($152). Neither number reflects the flat 3% rate on AGI of $23,006.18 less the $12,835 standard deduction, which gives $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.8-flash,reference_later_law,taxable_income_or_deductions,False,"It wrongly asserted that PolicyEngine does not model Louisiana income tax and returned 0. Louisiana's flat 3% tax on $10,171.18 of taxable income gives $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-4-argon,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction instead of the 2026 inflation-indexed $12,835, which gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,reference_later_law,taxable_income_or_deductions,False,"It got AGI and the flat 3% rate right but subtracted the 2025 $12,500 standard deduction instead of the 2026 indexed $12,835. The result was $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,glm-5.3,reference_later_law,taxable_income_or_deductions,False,"It taxed $10,506 at the repealed 1.85% bottom bracket instead of Louisiana's flat 3% rate. It also used the un-indexed $12,500 deduction instead of $12,835, so it got $194 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,reference_later_law,taxable_income_or_deductions,False,"It claimed taxable income falls below a lowest-bracket threshold. Louisiana has no zero bracket: after the $12,835 standard deduction, $10,171.18 is taxed at 3%, or $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,reference_later_law,taxable_income_or_deductions,False,"It assumed zero tax because wages are low. AGI of $23,006.18 exceeds the $12,835 standard deduction by $10,171.18, which is taxed at 3% to give $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 single standard deduction instead of the 2026 inflation-indexed $12,835, which gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,reference_later_law,taxable_income_or_deductions,False,"It applied the un-indexed $12,500 standard deduction instead of the 2026 amount of $12,835. That overstated taxable income by $335 and gave $315.18."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,reference_later_law,taxable_income_or_deductions,False,"It subtracted $12,500 instead of the 2026 inflation-indexed $12,835 single standard deduction, which gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,reference_later_law,taxable_income_or_deductions,False,"It subtracted the standard deduction plus a $4,500 personal exemption that the 2025 flat-tax reform repealed. It also used $12,500 instead of the indexed $12,835, which gave $180.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-astra,reference_later_law,taxable_income_or_deductions,False,"It correctly indexed the standard deduction but estimated it at $12,875 instead of the actual 2026 amount of $12,835. That understated taxable income by $40 and gave $303.93."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-luna,reference_later_law,taxable_income_or_deductions,False,"It used the 2025 $12,500 standard deduction instead of the 2026 inflation-indexed $12,835, which gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-6-sol,reference_later_law,taxable_income_or_deductions,False,"It subtracted the un-indexed $12,500 single deduction instead of $12,835, which overstated the tax by about $10."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-6.1-sol,reference_later_law,taxable_income_or_deductions,False,"It used $12,500 as the 2026 single standard deduction, but the inflation-indexed amount is $12,835. That gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-6.1-terra,reference_later_law,taxable_income_or_deductions,False,"It applied the $12,500 deduction instead of the 2026 indexed $12,835, so it taxed $10,506 instead of $10,171.18 at 3%."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,reference_later_law,taxable_income_or_deductions,False,"It claimed taxable income is zero or negative. The $12,835 standard deduction leaves $10,171.18 of the $23,006.18 AGI taxable at 3%, or $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,reference_later_law,taxable_income_or_deductions,False,"It used the repealed pre-2025 system: a $4,500 exemption and graduated 1.85%/3.5% brackets. It should have applied the flat 3% rate to AGI less the $12,835 standard deduction, so it got $441 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-4.6,reference_later_law,taxable_income_or_deductions,False,"It used the un-indexed $12,500 standard deduction instead of the 2026 amount of $12,835, which gave $315 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-4.7,reference_later_law,taxable_income_or_deductions,False,"It correctly noted the flat rate and the repeal of the federal tax deduction, but used $12,500 instead of the 2026 inflation-indexed $12,835 standard deduction. That gave $315.18."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,reference_later_law,taxable_income_or_deductions,False,"It subtracted the employer insurance premiums from gross wages, which put AGI at $14,617 instead of $23,006.18. It then used the repealed $4,500 deduction and the 1.85% bracket instead of the $12,835 deduction and the flat 3% rate."
+us,scenario_077,state_income_tax_before_refundable_credits,inkling,reference_later_law,taxable_income_or_deductions,False,"It used the $12,500 standard deduction rather than the 2026 indexed $12,835. That overstated taxable income by $335 and gave $315 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value or explanation for this output. The correct derivation is 3% × ($23,006.18 − $12,835) = $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,reference_later_law,taxable_income_or_deductions,False,"It subtracted $12,500 instead of the 2026 inflation-indexed single standard deduction of $12,835, which gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,reference_later_law,taxable_income_or_deductions,False,"It falsely stated that Louisiana has no state income tax. Louisiana levies a flat 3% tax on AGI less the $12,835 standard deduction, which is $305.14 here."
+us,scenario_077,state_income_tax_before_refundable_credits,muse-spark-1.3,reference_later_law,taxable_income_or_deductions,False,"It used the 2025 $12,500 deduction instead of the 2026 indexed $12,835, which gave $315.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,muse-wonder,reference_later_law,taxable_income_or_deductions,False,"It applied the un-indexed $12,500 single standard deduction. The 2026 amount is $12,835, which leaves $10,171.18 taxable and $305.14 of tax."
+us,scenario_077,state_income_tax_before_refundable_credits,ox-alpha,reference_later_law,taxable_income_or_deductions,False,"It subtracted the federal standard deduction of $16,100 plus a repealed $4,500 Louisiana exemption. Louisiana allows only its own $12,835 single standard deduction against AGI, so it got $72.18 instead of $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,reference_later_law,taxable_income_or_deductions,False,"It took the full $7,968 capital loss instead of the $3,000 limit, which put AGI at $18,038 instead of $23,006.18. It then used the repealed $4,500 exemption and 1.85%/3.5% brackets instead of the $12,835 standard deduction and the flat 3% rate."
+us,scenario_077,state_income_tax_before_refundable_credits,qwen3.8-max,reference_later_law,taxable_income_or_deductions,False,"It gave no computation. At 3%, its $347.24 implies about $11,575 of taxable income, which means a deduction of roughly $11,431 instead of Louisiana's 2026 single standard deduction of $12,835. The correct taxable income is $10,171.18 and the tax is $305.14."
us,scenario_077,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former 5% refundable EITC in tax year 2026. That state credit is unavailable for 2026, so multiplying an estimated federal EITC by 5% incorrectly produced $10.53 instead of zero."
us,scenario_077,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former refundable EITC rate of 5% to a purported $108 federal EITC. Louisiana provides no refundable EITC for tax year 2026, so the state refundable credit is zero."
us,scenario_077,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model incorrectly carried Louisiana’s former 5% refundable EITC into 2026 and also assigned a $625 federal EITC to a childless filer with $26,006 of earnings, above the childless-credit phaseout range. The unavailable state credit cannot produce the claimed $31.25."
us,scenario_077,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required structured-output contract."
us,scenario_077,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Put only the $26,460 real estate tax into SALT. The reference adds the engine's Maryland withholding estimate of state and county tax and reaches the $40,400 cap. It then made an addition error, turning its own $27,510 bracket sum into $33,499."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Included about $6,905 of Maryland state income tax in SALT but left out the county local income tax. That gave SALT of $33,365, when the reference's total, which uses the engine's Maryland withholding estimate of state and county tax, is above the $40,400 cap and the deduction should be the full $40,400. As a result, taxable income was $138,562 instead of $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Subtracted the $2,606 of unreimbursed employee business expenses from AGI, but these are not deductible. It also limited SALT to the $26,460 property tax. The reference adds the engine's Maryland withholding estimate of state and county tax and reaches the $40,400 cap."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Never computed the tax and submitted 0 as a placeholder. The reference calculation is $68,610 itemized ($40,400 SALT at the cap plus $28,210 interest), $131,526.91 taxable, and $24,164.46 tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Used the expired TCJA $10,000 SALT cap instead of the 2026 OBBBA cap of $40,400. This understated itemized deductions by $30,400 and gave taxable income of $161,927 instead of $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Used the old $10,000 SALT cap instead of the 2026 $40,400 cap, so itemized deductions came to $38,210 instead of $68,610. It then dropped its own $31,424 result for an unsupported $36,757."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at $10,000 instead of the 2026 OBBBA cap of $40,400. The reference fills the higher cap using property tax and the engine's Maryland withholding estimate of state and county tax, so the model's taxable income of $161,927 was $30,400.09 above the reference's $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only $6,936 of Maryland state income tax in SALT and left out the county piggyback income tax. That gave SALT of $33,396, while the reference's total uses the engine's Maryland withholding estimate of state and county tax and exceeds the $40,400 cap. Taxable income came out at $138,531 instead of $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Applied a $10,000 SALT cap instead of the 2026 cap of $40,400, which gave taxable income of $161,927. It also used invented inflation-adjusted brackets in place of the 2026 breakpoints ($12,400, $50,400 and $105,700)."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Used a $10,000 SALT cap instead of $40,400 and left the $1,632 refund out of AGI. It then inflated its own $31,069 result to $42,800 by citing Medicare surtax, which is not part of income tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Put only about $7,000 of Maryland state income tax into SALT and left out the county local income tax. The reference uses the engine's Maryland withholding estimate of state and county tax to reach the $40,400 cap and has taxable income of $131,526.91; the model calculated about $138,445."
-us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at $10,000 under the expired TCJA limit instead of the 2026 cap of $40,400. This left itemized deductions at $38,210 instead of $68,610."
-us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed TCJA expired and used pre-TCJA 10/15/25/28% brackets plus a $5,063 personal exemption. OBBBA kept the TCJA brackets for 2026 and no personal exemption exists."
-us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Applied pre-TCJA brackets and a $5,300 personal exemption, even though OBBBA kept the TCJA 10/12/22/24% structure for 2026. It also left Maryland county income tax out of SALT. The reference reaches the $40,400 cap using the engine's Maryland withholding estimate of state and county tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only the $26,460 property tax as SALT (the reference adds the engine's Maryland withholding estimate of state and county tax and reaches the $40,400 cap). That gave taxable income of $145,467 instead of $131,526.91, and it then misapplied the brackets as well."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed TCJA had expired, treating SALT as uncapped and applying a $5,150 personal exemption and pre-TCJA 15/25/28% brackets. Under OBBBA, 2026 keeps TCJA rates with a $40,400 SALT cap and no exemption. It also left the refund out of income."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Took a $15,000 standard deduction even though itemized deductions of $68,610 ($40,400 SALT plus $28,210 mortgage interest) are far larger. It also left the $1,632 refund out of AGI."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed TCJA expired, bringing back personal exemptions and uncapped SALT and adding an AMT. In fact OBBBA kept TCJA brackets for 2026 with a $40,400 SALT cap, and no AMT applies at $131,527 of taxable income."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed TCJA expired and applied a $5,150 personal exemption with pre-TCJA brackets instead of the 2026 TCJA brackets kept by OBBBA. Its SALT figure also included only about $6,886 of state income tax and no county tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Used the standard deduction when itemizing is larger; the reference itemizes $68,610 ($40,400 SALT at the cap plus $28,210 mortgage interest). That overstated taxable income by about $52,500."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Treated SALT as uncapped and applied a $5,050 personal exemption with pre-TCJA brackets, assuming TCJA expired. OBBBA kept the 2026 TCJA brackets with no exemption and capped SALT at $40,400."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no derivation. Its $31,316 is consistent with itemizing under the old $10,000 SALT cap (taxable income about $162,000), not the 2026 $40,400 cap that gives $131,526.91 taxable."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Explicitly applied TCJA-expiration rules for 2026. OBBBA made the TCJA brackets permanent and set a $40,400 SALT cap, which the reference fills using property tax and the engine's Maryland withholding estimate of state and county tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"Limited SALT to the $26,460 real estate tax. The reference adds the engine's Maryland withholding estimate of state and county tax and reaches the $40,400 cap. It then mis-taxed its own $145,467 taxable income, which at 2026 brackets is $27,510, not $32,145."
-us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Took a $15,000 standard deduction instead of itemizing $68,610. It also invented a $12,000 child and dependent care credit, although the filer has no dependents or care expenses."
-us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the $10,000 TCJA SALT cap instead of the 2026 OBBBA cap of $40,400. That understated itemized deductions by $30,400."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"Applied a $10,000 SALT cap and mixed up standard and itemized deductions. Its $42,034 is higher than the $36,767 tax under the standard deduction, which no correct deduction choice could produce. The reference itemized total is $68,610."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized only mortgage interest and property tax. The reference adds the engine's Maryland withholding estimate of state and county tax and reaches the $40,400 cap. It also miscomputed the tax on the resulting income."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only the $26,460 property tax as SALT. The reference adds the engine's Maryland withholding estimate of state and county tax and reaches the $40,400 cap. It also mis-taxed its own $145,467, which is $27,510.08 at 2026 brackets."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its result implies SALT of about $33,000, meaning it added only part of Maryland state income tax to property tax and left out the county local income tax. The reference reaches the $40,400 cap using the engine's Maryland withholding estimate of state and county tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Used $26,460 of real estate tax for SALT and calculated $145,467 of taxable income. The reference adds the engine's Maryland withholding estimate of state and county tax, reaches the $40,400 cap, and has $131,526.91 of taxable income."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized only mortgage interest and real estate tax, using $26,460 of SALT. The reference adds the engine's Maryland withholding estimate of state and county tax and reaches the $40,400 cap."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Stated that no other deductible tax payments were reported and used $26,460 of real estate taxes as SALT. The reference adds the engine's Maryland withholding estimate of state and county income tax, which raises SALT above the $40,400 cap; actual tax withheld is unlisted in the prompt."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized only $26,460 of property tax as SALT. The reference adds the engine's Maryland withholding estimate of state and county tax, reaches the $40,400 cap, and has about $13,940 less taxable income than the model."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Added about $6,936 of Maryland state income tax to SALT. The reference uses the engine's Maryland withholding estimate of state and county tax to reach the $40,400 cap and has taxable income of $131,526.91, about $7,000 below the model's approximately $138,531."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Included Maryland state income tax in SALT but not the county local income tax, so SALT stayed below the $40,400 cap. The reference reaches the $40,400 cap using the engine's Maryland withholding estimate of state and county tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Deducted only real estate tax as SALT (the reference adds the engine's Maryland withholding estimate of state and county tax and reaches the $40,400 cap). That gave taxable income of $145,467 instead of $131,526.91."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no reasoning. Its $41,500 is higher than the $36,767 tax under the standard deduction, which means it ignored both the $68,610 itemized deduction and the 2026 brackets."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Applied a $5,427 personal exemption and post-sunset brackets, but OBBBA kept TCJA rates for 2026 with no exemption. It also left Maryland state and county income tax out of SALT. The reference reaches the $40,400 cap using the engine's Maryland withholding estimate of state and county tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Used restored 10/15/25/28% brackets and a $5,300 personal exemption on the assumption that TCJA expired. OBBBA kept the 2026 TCJA brackets, and the reference reaches the $40,400 SALT cap using the engine's Maryland withholding estimate of state and county tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Limited SALT to the $26,460 property tax. The reference adds the engine's Maryland withholding estimate of state and county tax and reaches the $40,400 cap. It also used the wrong lower bracket breakpoints instead of $12,400 and $50,400."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed TCJA expired, applying a $5,434 personal exemption and pre-TCJA brackets. Under OBBBA, 2026 uses TCJA brackets with no exemption and a $40,400 SALT cap."
-us,scenario_078,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"Treated SALT as property tax only ($26,460) and taxed approximately $145,467. The reference adds the engine's Maryland withholding estimate of state and county tax, reaches the $40,400 cap, and has $131,526.91 of taxable income."
-us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,Did not return a parseable value for federal_income_tax_before_refundable_credits.
-us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly identified the $40,400 SALT cap but filled it only with the $26,460 property tax. It calculated taxable income of $145,467 using the reported property tax. The reference reaches the full cap by adding the engine's Maryland withholding estimate of state and county tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"Submitted 0 without computing anything. The reference has $131,526.91 of taxable income after $68,610 of itemized deductions, which produces $24,164.46 of tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"Limited SALT to property tax. The reference reaches the $40,400 cap by adding the engine's Maryland withholding estimate of state and county tax. It also blended 2025 and 2026 brackets."
-us,scenario_078,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized only $26,460 of property tax as SALT. The reference adds the engine's Maryland withholding estimate of state and county tax, reaches the $40,400 cap, and has about $13,940 less taxable income than the model."
-us,scenario_078,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the higher 2026 SALT cap but counted only the $26,460 real estate tax against it. The reference reaches the $40,400 cap by adding the engine's Maryland withholding estimate of state and county tax to property tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at the expired $10,000 TCJA limit instead of the 2026 cap of $40,400. It also used 2024 brackets instead of the 2026 breakpoints."
-us,scenario_078,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"Invented an AMT liability by treating itemized deductions as an AMT preference and subtracting a made-up foreign-tax offset. No AMT applies here. It also left Maryland state and county income tax out of SALT. The reference reaches the $40,400 cap using the engine's Maryland withholding estimate of state and county tax."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $26,460 real estate tax as SALT and left out Maryland state and county income tax, which push SALT to the $40,400 cap. That understated itemized deductions by $13,940. It then got $27,510 for its own taxable income of $145,467 but mis-added the brackets and reported $33,499."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It added only about $6,905 of Maryland state income tax to SALT and left out the Maryland county income tax. Its SALT of $33,365 is therefore below the $40,400 cap, when the full state-plus-local tax makes the cap bind. Itemized deductions came out $7,035 too low, and the tax was overstated at the 24% rate."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated Maryland income tax at $9,743.53 and left SALT at $36,203.53. Maryland state plus county income tax on about $200k is at least $13,940, so SALT reaches the $40,400 cap. Itemized deductions were understated by about $4,196."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the $2,606 of unreimbursed employee business expenses from AGI, but these are not deductible at all, either above the line or as an itemized deduction. It also left Maryland state and county income tax out of SALT, which should be capped at $40,400. Its tax of $25,779 on $142,861 of taxable income does not match the 2026 brackets either."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly declined to compute the tax and entered 0. With $68,610 of itemized deductions, the $131,526.91 of taxable income produces $24,164.46 at the 2026 single brackets."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the pre-OBBBA $10,000. For 2026 the cap is $40,400, so the full cap applies to real estate tax plus Maryland state and local income tax. Itemized deductions were understated by $30,400, and it also used a wrong $107,450 top for the 22% bracket."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 cap of $40,400, which gave itemized deductions of $38,210 instead of $68,610. It then dropped its own $31,424 result for an unsupported $36,757, which is about what the standard deduction would produce."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the old $10,000 SALT cap. The 2026 cap is $40,400, and real estate tax plus Maryland state and county income tax fill it. That left taxable income at $161,927 instead of $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT of $33,396 included only $6,936 of Maryland state income tax and left out the Maryland county income tax. Including the county tax pushes SALT past the $40,400 cap. Itemized deductions were $7,004 too low, which added about $1,681 of tax at 24%."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 cap of $40,400, which understated itemized deductions by $30,400. It also used guessed inflation-indexed brackets instead of the actual 2026 thresholds of $12,400, $50,400 and $105,700."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It used a $10,000 SALT cap instead of $40,400 and left the $1,632 state refund out of income. It then raised its own $31,069 result to $42,800 by wrongly folding in Medicare surtaxes, which are not part of income tax."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only about $7,000 of Maryland state income tax in SALT and left out the county income tax. Its SALT of about $33,460 should have been the full $40,400 cap, so itemized deductions were understated by about $7,000."
+us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It capped property taxes at the pre-OBBBA $10,000 SALT limit. In 2026 the cap is $40,400, and real estate tax plus Maryland state and local income tax fill it. Taxable income came out at $161,927 instead of $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired, applying a $5,063 personal exemption and the pre-TCJA 15/25/28% brackets. OBBBA made the TCJA rates permanent, so 2026 uses the 10/12/22/24% schedule with no personal exemption. Its uncapped SALT also exceeds the $40,400 cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied a $5,300 personal exemption and the pre-TCJA brackets, but OBBBA kept the 10/12/22/24% schedule and eliminated exemptions permanently. It also put only $7,472 of Maryland income tax in SALT instead of reaching the $40,400 cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized deductions of $54,670 counted only real estate tax in SALT and left out Maryland state and county income tax, which bring SALT to the $40,400 cap. Taxable income was overstated by $13,940."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired, with uncapped SALT, a $5,150 personal exemption, and 15/25/28% brackets. In 2026 the 10/12/22/24% schedule applies, there is no exemption, and SALT is capped at $40,400. It also left the $1,632 state refund out of income."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction and ignored $68,610 of itemized deductions ($40,400 SALT plus $28,210 mortgage interest), which far exceed the $16,100 standard deduction. It also left out the taxable $1,632 refund."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired, with personal exemptions, no SALT cap, and an AMT trigger. Under OBBBA, SALT is capped at $40,400 and the 24% bracket applies. AMTI of about $171,927 less the $90,100 exemption gives a tentative minimum tax below regular tax, so no AMT arises."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset law with a $5,150 personal exemption and pre-TCJA brackets instead of the permanent 10/12/22/24% schedule. It also counted only about $6,886 of state income tax, so SALT fell short of the $40,400 cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It used the standard deduction even though itemized deductions of $68,610 are far larger. Even on that basis, its $33,020 is below the $36,767 the 2026 brackets produce with the standard deduction."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired, leaving SALT uncapped at $37,999 and adding a $5,050 personal exemption under pre-TCJA brackets. In 2026 there is no exemption, SALT is capped at $40,400, and the 10/12/22/24% schedule applies."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $31,316 matches itemizing with SALT capped at $10,000, which gives about $31,460 on $161,927 of taxable income. The 2026 cap is $40,400, and real estate tax plus Maryland state and local income tax fill it, for a total of $68,610 in itemized deductions."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-expiration rules, but OBBBA kept the 2026 10/12/22/24% brackets and the $40,400 SALT cap. It also counted only real estate tax, leaving out Maryland state and county income tax."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"It left Maryland state and county income tax out of SALT, which made itemized deductions $54,670 instead of $68,610. It then also misapplied the brackets: $145,467 of taxable income gives $27,510, not $32,145."
+us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction instead of the $68,610 of itemized deductions and left the refund out of income. It then subtracted an invented $12,000 child and dependent care credit, even though there are no children or care expenses."
+us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 cap of $40,400, so itemized deductions came to $38,210 instead of $68,610. It also used a wrong $107,250 top for the 22% bracket."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It applied a $10,000 SALT cap and mixed in the standard deduction. Its $42,034 exceeds even the $36,767 that the standard deduction produces, so it both understated deductions and overstated rates."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only mortgage interest and real estate tax and left out Maryland state and county income tax. Including them brings SALT to the $40,400 cap and taxable income to $131,526.91. Its $28,347 also does not match the 2026 brackets for its own deductions."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $26,460 real estate tax in SALT and left out Maryland state and county income tax, which bring SALT to the $40,400 cap. It also misapplied the brackets: $145,467 of taxable income gives $27,510.08, not $29,576.96."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $25,947.40 implies a SALT deduction of only about $33,000, which means it left out part of the Maryland state and county income tax. With that tax included, SALT reaches the $40,400 cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted only real estate tax as SALT and left out Maryland state and county income tax, which bring SALT to the $40,400 cap. Taxable income came out at $145,467 instead of $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only mortgage interest and real estate tax and left out Maryland state and county income tax. Including them makes SALT the capped $40,400, so taxable income was overstated by $13,940."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It stated that no other deductible tax payments were reported and so left out Maryland state and county income tax withheld on $198,505 of wages. That tax is deductible SALT and brings the total to the $40,400 cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It left Maryland state and county income tax out of SALT and itemized only $54,670. The full SALT deduction is the $40,400 cap, for $68,610 of itemized deductions."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It added $6,936.34 of Maryland state income tax but left out the Maryland county income tax. Its SALT therefore stayed below the $40,400 cap that the full state-plus-local amount reaches."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated only Maryland state income tax and left out the county piggyback income tax. SALT therefore stayed at about $33,400 instead of the $40,400 cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted only mortgage interest and real estate tax and left out Maryland state and county income tax. With those included, SALT reaches $40,400 and taxable income falls to $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"With no reasoning given, its $41,500 exceeds even the $36,767 tax under the standard deduction. That means it ignored the $68,610 of itemized deductions and also overstated the bracket rates."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired, adding a $5,427 personal exemption and pre-TCJA brackets, but OBBBA kept the 2026 10/12/22/24% schedule with no exemption. It also left Maryland state and county income tax out of SALT."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It applied restored 10/15/25/28% brackets and a $5,300 personal exemption, but 2026 uses the permanent TCJA/OBBBA rates. It also counted only $6,936 of Maryland state tax and left out the county tax, so SALT fell short of the $40,400 cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only real estate tax as SALT and left out Maryland state and county income tax, which bring SALT to the $40,400 cap. It also used wrong 10% and 12% bracket thresholds."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA brackets and a $5,434 personal exemption, but OBBBA made the 10/12/22/24% schedule permanent and eliminated exemptions. Its uncapped SALT of $36,346 also ignores the $40,400 cap logic: the full state-plus-local tax reaches the cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It treated $26,460 of real estate tax as the whole SALT deduction and left out Maryland state and county income tax. With those included, SALT reaches the $40,400 cap and itemized deductions are $68,610."
+us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,No parseable value was returned for federal_income_tax_before_refundable_credits.
+us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly applied the $40,400 SALT cap but filled it only with the $26,460 real estate tax. It left out Maryland state and county income tax, which bring SALT to the full cap, so taxable income was overstated by $13,940."
+us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It returned 0 with no computation. Taxable income of $131,526.91 after $68,610 of itemized deductions produces $24,164.46 at the 2026 single brackets."
+us,scenario_078,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only real estate tax as SALT and left out Maryland state and county income tax, which bring SALT to the $40,400 cap. It also used a mix of 2025 and 2026 brackets."
+us,scenario_078,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only $26,460 of property tax as SALT and left out Maryland state and county income tax. With those included, SALT is the capped $40,400 and taxable income is $131,526.91."
+us,scenario_078,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It recognized the higher 2026 SALT cap but counted only real estate tax under it. It left out Maryland state and county income tax, which bring SALT to the $40,400 cap."
+us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 cap of $40,400 and then applied 2024 brackets instead of the 2026 schedule. Itemized deductions came out $30,400 too low."
+us,scenario_078,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It ended a 24% bracket at about $135,950 and pushed income into 32%, but the 2026 24% bracket runs to $201,775. It also invented an AMT liability. It left Maryland income tax out of SALT as well. With SALT at $40,400, AMTI less the $90,100 exemption gives a tentative minimum tax below regular tax."
us,scenario_078,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_078,local_income_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model incorrectly treated Maryland county income tax as part of the requested local_income_tax variable and applied an assumed 3.03% average county rate to taxable income. This output excludes Maryland county tax and only captures the specifically listed modeled city taxes, none of which applies to this household."
us,scenario_078,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning gets the correct total of $14,317.32 ($11,439 SS on the $184,500 base plus $2,878.32 Medicare), but it submitted $12,240.44. Nothing in its derivation supports that figure, so the submitted value contradicts its own correct computation."
@@ -6899,44 +6973,45 @@ us,scenario_078,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,Fals
us,scenario_078,payroll_tax,minimax-m3,llm_error,missing_output,False,"It entered 0 even though the household has $198,505 in wages, which leaves out all employee FICA. The correct answer is 6.2% Social Security on $184,500 ($11,439) plus 1.45% Medicare on all wages ($2,878.32)."
us,scenario_078,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"It projected the 2026 wage base as $174,900 by inflating the 2024 figure, instead of using the actual 2026 base of $184,500. That understates SS tax by $595.20 ($10,843.80 vs $11,439)."
us,scenario_078,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It capped Social Security at the 2024 base of $168,600 ($10,453.20) instead of the 2026 base of $184,500. It then submitted $15,622.81, which does not match its own components ($10,453.20 + $2,878.32 = $13,331.52)."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used a roughly $2,700 standard deduction and an $800 exemption, which put taxable income near $195,005. It never itemized the $54,670 of mortgage interest and uncapped property tax. The exemption is $0 above $150,000, and the correct taxable income is $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It got Maryland AGI ($198,505) and the itemized deductions right. It then subtracted a $600 personal exemption, but the single-filer exemption is fully phased out to $0 above $150,000 AGI. That understated taxable income by about $600. It also skipped the $10.27 reduction for 7.5% of AGI over $200,000."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It took a $3,200 standard deduction and taxed about $195,305. It never itemized the $28,210 of mortgage interest and $26,460 of property tax, which Maryland allows without the federal SALT cap. Correct taxable income is $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It said outright that it had not computed Maryland income tax and submitted $0. The correct calculation taxes $143,845.27 of Maryland taxable income, which gives $6,936.88."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its own working reached taxable income of about $143,835 and tax of $6,936.59, which matches the correct method. It then dropped that result and submitted $9,755 with no supporting calculation. That figure matches taxing about $194,000, as if the itemized deductions had never been taken."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It wrongly applied the federal $10,000 SALT cap to the Maryland itemized deduction, which cut the deduction to about $30,000 and gave $8,277.78. Maryland allows the full $26,460 of property tax. It then submitted $9,876, which matches the standard-deduction path on about $197,500 of taxable income and contradicts its own working."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It capped property tax at $10,000, giving an itemized total of $38,210 instead of $54,670. It also started from $200,137 of federal AGI without subtracting the $1,632 refund, which produced taxable income of about $161,900 instead of $143,845.27. Its stated rate structure (5.0% for $150k–$175k) is also wrong."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It took the standard deduction instead of itemizing $54,670 of mortgage interest and property tax. It kept the $1,632 refund in Maryland AGI and claimed a $3,200 exemption that is $0 above $150,000. Together these gave taxable income of $194,537 instead of $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It reached taxable income of about $143,835. It then applied a vague 'effective near 5.0%' rate and got $9,200. Applying the actual brackets ($90 on the first $3,000, 4.75% to $100k, 5% to $125k, 5.25% above) gives about $6,937."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used $200,137 as Maryland AGI and missed that Maryland subtracts the $1,632 state tax refund, which brings AGI to $198,505. Taxable income came out at $145,467 instead of $143,845.27, so the tax was about $85 too high."
-us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It capped property tax at the federal $10,000 SALT limit, but Maryland allows the full $26,460. It also claimed a $3,200 exemption that is $0 above $150,000 AGI and left the $1,632 refund in AGI. That gave taxable income of $158,727 instead of $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It took a $2,400 standard deduction and a $3,200 exemption on $200,137 of AGI. It skipped $54,670 of itemized deductions, kept the refund in AGI, and claimed an exemption that has phased out at this income. That gave taxable income of $194,537 instead of $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It correctly subtracted the refund to reach $198,505. It then cut itemized deductions by about $10,027, down to $44,642.60. The actual limitation is 7.5% of federal AGI above $200,000, which is only $10.27 here and leaves $54,659.73 of deductions."
-us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It made three errors. It kept the $1,632 refund in Maryland AGI ($200,137 instead of $198,505). It used a 3% limitation ($4.11) instead of 7.5% of AGI over $200,000 ($10.27). It also claimed a $1,600 exemption that is $0 above $150,000. These partly cancelled, leaving taxable income $25.84 too high at $143,871.11."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no working. Under Maryland's brackets, $9,070 matches taxable income of about $183,000, roughly wages minus a federal-sized standard deduction. It never subtracted the $54,670 of uncapped mortgage interest and property tax that bring taxable income down to $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It added county/local income tax into the figure, but this output is state income tax only. State tax on $143,845.27 of taxable income is $6,936.88, and the county piece accounts for the extra $4,400 or so."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It got taxable income right at about $143,835 with a $0 exemption. It then misapplied the brackets and came out about $50 low. The correct sum is $90 + $4,607.50 (4.75% to $100k) + $1,250 (5% to $125k) + 5.25% on the remainder, about $6,936."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It explicitly used the standard deduction. $9,876 matches taxing about $197,500 at the 5.5% top bracket. It never took the $54,670 of itemized mortgage interest and uncapped property tax that bring taxable income down to $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It computed the state tax correctly at $6,936.34. It then added a 3.2% county income tax ($4,602.72), even though this output excludes local income tax."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the right deductions (mortgage interest plus property tax, $0 exemption). Its bracket arithmetic came out about $48 short of the $6,936.88 that the 2/3/4/4.75/5/5.25% schedule gives on $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"It got taxable income right at about $143,835. It then included county tax as part of state tax, adding about $3,236 (2.25% of taxable income). State income tax alone is $6,936.88."
-us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It left the $1,632 state refund in Maryland AGI, giving taxable income of $145,467 instead of $143,845.27. It also misapplied the brackets: correct rates on $145,467 give about $7,022, not $7,300.69."
-us,scenario_078,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It started from the SALT-capped federal itemized total ($38,210) and added back state income tax in a circular calculation at a flat 4.75%. Maryland allows the full $26,460 of property tax, no state income tax was deducted, and the correct deduction is $54,659.73 against $198,505 of Maryland AGI."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It used the 'standard allowance' instead of itemizing $54,670 of mortgage interest and property tax. $9,720 matches taxing about $195,000 rather than the correct $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Under Maryland's brackets, $10,120 matches taxable income of about $202,000. That is essentially all income taxed with no deductions. It never applied the $54,659.73 of itemized deductions or the refund subtraction that bring taxable income to $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its answer implies taxable income of about $135,700, roughly $8,100 below the correct $143,845.27. It claimed a personal exemption that is $0 above $150,000 AGI and deducted more than the mortgage interest and property tax allow."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"$7,025 matches taxable income of about $145,500. That means it left the $1,632 state refund in Maryland AGI instead of subtracting it to reach $198,505 and taxable income of $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used taxable income of $145,467, which leaves the $1,632 refund in Maryland AGI. Maryland subtracts the refund, giving AGI of $198,505 and taxable income of $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It gave no working. Under Maryland's brackets, $9,200 matches taxable income of about $185,000. That means it never deducted the $54,670 of uncapped mortgage interest and property tax that bring taxable income down to $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It took the $2,400 standard deduction on $200,137 of AGI and taxed $197,737. It missed $54,670 of itemized mortgage interest and property tax and did not subtract the $1,632 refund."
-us,scenario_078,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It reduced itemized deductions by 3% of AGI over $100,000 ($3,004); the actual limitation is 7.5% of AGI over $200,000 ($10.27). It also kept a $576 exemption that is fully phased out above $150,000 and left the refund in AGI. That gave taxable income of $147,895 instead of $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no parseable value for state_income_tax_before_refundable_credits, so there is no substantive answer to evaluate. The correct value comes from taxing $143,845.27 of Maryland taxable income."
-us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,missing_output,False,"It submitted $0 with no calculation. A filer with $198,505 of Maryland AGI and $143,845.27 of taxable income owes $6,936.88 under the state brackets."
-us,scenario_078,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It deducted only the $28,210 of mortgage interest and left out the $26,460 of real estate tax, which Maryland allows uncapped. It also claimed a $3,200 exemption that is $0 above $150,000 and kept the refund in AGI, giving taxable income of $168,727."
-us,scenario_078,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It subtracted a reduced personal exemption of about $800, putting taxable income at $143,035, but the exemption is $0 above $150,000 AGI. It also taxed income 'mainly at 4.75%' and underused the 5% and 5.25% brackets above $100k and $125k, which made the tax too low."
-us,scenario_078,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It applied a flat 4.75% to everything above $3,000. It ignored the 5% bracket from $100k to $125k and the 5.25% bracket above $125k. It also left the $1,632 refund in Maryland AGI, which gave taxable income of $145,467 instead of $143,845.27."
-us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It ignored the $54,670 of itemized mortgage interest and property tax and taxed about $196,000 after a standard deduction. It then reported $10,933, which is higher than its own no-deduction figure of $9,927.78."
-us,scenario_078,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from federal taxable income ($184,037) instead of Maryland AGI ($198,505) minus Maryland itemized deductions ($54,659.73). It used a made-up bracket schedule and subtracted a nonrefundable credit that no listed fact supports."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Used a capped Maryland standard deduction of about $2,700 and an $800 exemption, which left about $195k taxed into the 5.5% bracket. It never itemized the $54,670 of mortgage interest and property taxes. Maryland itemized deductions bring taxable income down to $143,845.27, which tops out in the 5.25% bracket."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Claimed a $600 personal exemption, but a single filer's exemption is $0 once Maryland AGI is above $150,000. It also skipped the $10.27 itemized-deduction reduction (7.5% of federal AGI over $200,000). Together these understated taxable income by $610.27 and the tax by $32.04."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"Did not apply Maryland's subtraction for the $1,632 state tax refund included in federal AGI, so it started from $200,137 instead of $198,505. Its taxable income was $1,632 too high, adding $85.68 at the 5.25% rate. The rest of its calculation, including the 7.5% itemized phase-out, matched."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Used a Maryland standard deduction of about $3,200 and taxable income of about $195,305 instead of itemizing $28,210 of mortgage interest plus the full $26,460 of property taxes. It also mistakenly treated the federal $10,000 SALT cap as limiting the deduction. Its $7,558 doesn't even follow from the brackets applied to its own taxable income."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"Said outright that it did not compute Maryland income tax and submitted 0. The correct calculation (Maryland AGI $198,505 minus $54,659.73 of itemized deductions, run through the brackets) gives $6,936.88."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its own working reached taxable income of about $143,835 and tax of $6,936.59, essentially the correct result. It then dropped that figure and submitted an unsupported $9,755, which is consistent with taxing about $194k without itemized deductions."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Treated Maryland itemized deductions as limited by the federal $10,000 SALT cap and assumed about $30,000 in total, leaving taxable income of $168,505. Maryland allows the full $26,460 of property taxes plus mortgage interest. It then submitted $9,876, which doesn't match even its own $8,277.78 calculation."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Capped real estate taxes at $10,000, giving about $38,210 of itemized deductions instead of $54,659.73. It also did not subtract the $1,632 state tax refund from federal AGI. The result was taxable income of about $161,900 instead of $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Took a $2,400 standard deduction and a full $3,200 exemption on federal AGI of $200,137. It should have itemized $54,659.73 (mortgage interest plus uncapped property taxes, after the 7.5% phase-out), subtracted the $1,632 refund, and used a $0 exemption above $150,000. Its taxable income of $194,537 reached the 5.5% bracket instead of stopping at $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Got taxable income of about $143,835, close to the correct $143,845.27, but then estimated about $9,200 from a vague effective-rate assumption. Applying the actual brackets (2/3/4% on the first $3,000, 4.75% to $100k, 5% to $125k, 5.25% above that) gives about $6,937."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Started Maryland AGI at the federal $200,137 and missed the Maryland subtraction for the $1,632 state tax refund. That left taxable income of $145,467 instead of $143,845.27, overstating the tax by about $85 at the 5.25% rate."
+us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Capped property taxes at $10,000, although Maryland allows the full $26,460, and claimed a $3,200 exemption that is $0 above $150,000. It also skipped the $1,632 refund subtraction. Taxable income came out at $158,727 instead of $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Used a $2,400 standard deduction and a $3,200 exemption on $200,137 instead of itemizing $54,659.73 of mortgage interest and property taxes from Maryland AGI of $198,505. Its taxable income of $194,537 pushed income into the 5.5% bracket."
+us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Started correctly from Maryland AGI of $198,505 but cut itemized deductions by about $10,027, to $44,642.60. Maryland reduces them only by 7.5% of federal AGI above $200,000, which here is $10.27, leaving $54,659.73. Its taxable income of $153,862 was about $10k too high."
+us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Three errors nearly cancelled out. It did not subtract the $1,632 refund from Maryland AGI, it applied a 3% itemized phase-out instead of 7.5%, and it claimed a $1,600 personal exemption that is $0 above $150,000. The result was taxable income of $143,871.11 instead of $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Gave no derivation. Its $9,070 is consistent with taxing roughly $180k+ into the 5.5% bracket, which means it omitted most of the $54,659.73 of Maryland itemized deductions (mortgage interest plus uncapped property taxes). The correct taxable income is $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Added county income tax on top of the state tax, although the requested output covers only the state tax and excludes local tax. The state-only tax on taxable income of $143,845.27 is $6,936.88."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"Correctly itemized and used a $0 exemption, reaching taxable income of $143,835. It then misapplied the brackets and got $6,886; the 2/3/4/4.75/5/5.25% schedule gives about $6,936 on that income. It also skipped the $10.27 itemized phase-out."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Said it used standard deductions and exemptions. Its $9,876 is consistent with about $197.5k of taxable income reaching the 5.5% bracket, which means it never itemized the $54,670 of mortgage interest and property taxes."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Computed the state tax at $6,936.34, essentially correct, then added 3.2% county income tax ($4,602.72). The requested output excludes local income tax."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Correctly itemized mortgage interest and property taxes with a $0 exemption but made a bracket arithmetic slip, coming in $48 short. Taxable income of $143,845.27 through the 2/3/4/4.75/5/5.25% brackets gives $6,936.88."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"Correctly reached taxable income of about $143,835 but explicitly folded county income tax (about 2.25%) into the state figure. The requested output excludes local income tax, and the state-only amount is $6,936.88."
+us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Did not subtract the $1,632 state tax refund, so its taxable income was $145,467. It then miscomputed the brackets, getting $7,300.69 when that income actually yields about $7,022. Correct taxable income is $143,845.27, which gives $6,936.88."
+us,scenario_078,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Built Maryland itemized deductions from a federal figure with a $10,000 SALT cap ($38,210), then added a circular state-income-tax addback. Maryland allows mortgage interest plus the full $26,460 of property taxes ($54,670 before the $10.27 phase-out). Its approach inflated taxable income well above $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Used a Maryland standard allowance. Its $9,720 is consistent with about $194.7k of taxable income in the 5.5% bracket, so it ignored the $54,659.73 of itemized mortgage interest and property taxes that bring taxable income to $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Gave no calculation. Its $10,120 matches taxing roughly $202k, essentially full wages with no deductions, through the 5.5% bracket. Subtracting the refund and $54,659.73 of itemized deductions leaves $143,845.27, which tops out at 5.25%."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Applied a personal exemption on top of itemized deductions, although a single filer's exemption is $0 above $150,000 of Maryland AGI. It understated taxable income by about $8,100 relative to $143,845.27, producing $6,511.18."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $7,025 matches taxable income of about $145,467, meaning it itemized correctly but did not subtract the $1,632 state tax refund. That overstated the tax by about $85 at the 5.25% rate."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Used taxable income of $145,467, which starts from federal AGI without Maryland's subtraction for the $1,632 state tax refund. Correct taxable income is $143,845.27, which gives $6,936.88."
+us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Gave no reasoning. Its $9,200 is consistent with about $185k of income taxed into the 5.5% bracket, meaning it omitted most of the $54,659.73 of Maryland itemized deductions. The correct taxable income is $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Took a $2,400 standard deduction from $200,137, giving taxable income of $197,737. It should have subtracted the $1,632 refund and itemized $54,659.73 of mortgage interest and uncapped property taxes, leaving $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Applied a 3% reduction on AGI over $100,000 ($3,004); Maryland's limit is 7.5% of federal AGI over $200,000, which here is $10.27. It also kept a $576 exemption that should be $0 and skipped the $1,632 refund subtraction. Taxable income came out at $147,895 instead of $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no parseable value for state_income_tax_before_refundable_credits. The correct derivation gives $6,936.88 on taxable income of $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,missing_output,False,"Submitted 0 with no derivation, effectively treating the household as owing no Maryland tax. Maryland AGI of $198,505 minus $54,659.73 of itemized deductions leaves $143,845.27 of taxable income and $6,936.88 of tax."
+us,scenario_078,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"Itemized only the $28,210 of mortgage interest and left out the $26,460 of real estate taxes, which Maryland allows in full. It also claimed a $3,200 exemption that is $0 above $150,000 and skipped the $1,632 refund subtraction. Taxable income came out at $168,727 instead of $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"Taxed everything above $3,000 at a flat 4.75%, skipping the 5% bracket ($100k–$125k) and the 5.25% bracket above $125k. Its $6,741.66 equals $90 + 4.75% × $140,035. It also claimed a reduced exemption, which should be $0 above $150,000."
+us,scenario_078,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Taxed all income above $3,000 at a flat 4.75% ($90 + 4.75% × $142,467), skipping the 5% and 5.25% brackets above $100k. It also did not subtract the $1,632 state tax refund from Maryland AGI."
+us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Assumed no Maryland itemized deductions and taxed about $196k of wages into the 5.5% bracket. Its $10,933 even exceeds the $9,927.78 it computed with no deductions at all. It ignored the $54,659.73 of itemized mortgage interest and property taxes that bring taxable income to $143,845.27."
+us,scenario_078,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Started from federal taxable income and subtracted a Maryland standard deduction; Maryland actually starts from AGI ($198,505 after the refund subtraction) and allows $54,659.73 of itemized deductions. It also used a made-up bracket schedule and an unexplained nonrefundable credit."
us,scenario_078,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_079,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model tested only Arizona's income-based adult (MAGI) Medicaid rule and declared the head failed it, never applying the SSI-recipient category: the head's blindness, disability, and $7,560 of SSDI produce $5,286 of SSI, and Arizona is a 1634 state where SSI approval automatically confers AHCCCS/Medicaid with no separate income or resource test. Its own stated route also cannot yield 'No' — countable income at 0.35 x FPL sits far below Arizona's 138% FPL expansion limit — so the model both skipped the operative categorical pathway and misapplied the fallback one it invoked."
us,scenario_079,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated Medicaid eligibility as requiring an explicitly listed pathway or coverage input and returned 0 for lack of one, but SSI receipt is computed from the facts given (age 58, blind, disabled, $7,560 SSDI, $1,045 in resources, no other income), and that SSI receipt is itself the categorical Medicaid pathway in Arizona, which auto-enrolls SSI recipients. It compounded this by reading Indian Health Service coverage at interview as the head's coverage source; IHS is not minimum essential coverage and never displaces or disqualifies Medicaid eligibility."
@@ -6969,57 +7044,58 @@ us,scenario_079,head_medicare_eligible,muse-wonder,prompt_ambiguity,age_disabili
us,scenario_079,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,"The model stated the 24-month rule, then read 'facts constant throughout the year' as meeting it. Constant receipt for one year does not establish 24 months of entitlement, and the unlisted months-of-receipt input is 0, so the head is ineligible."
us,scenario_079,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_079,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Counted only the $630/month of SSDI and left out the $881/month of SSI the couple receives, so gross income was understated by more than half. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, and its final $4,560 doesn't follow from its own arithmetic."
-us,scenario_079,snap,claude-fable-5.1,llm_error,state_local_rule,False,"Got gross income ($1,511), the $209 standard deduction and the $546 maximum right, but deducted only the $26 actual medical excess. Arizona's $145 standard medical deduction applies instead, which makes net income $1,157 rather than $1,276 and the benefit $198/month rather than ~$163."
-us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"Ruled the household out on assets by counting vehicle equity and home equity. SNAP excludes the home and these vehicles, and the household is categorically eligible anyway. Countable resources are just the $1,045 bank account, which passes, so the household gets $198/month."
-us,scenario_079,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"Said income is above the gross limit, but $1,511/month is only 86% of the poverty guideline, and net income of $1,157 passes the net test. The net-income formula gives $546 − $348 = $198/month, not zero."
-us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"Left the $881/month of SSI out of gross income and counted only $630 of SSDI. It used a $219 standard deduction and the $26 actual medical excess instead of $209 and Arizona's $145 standard medical deduction, then cut its own $420.50 result to an unexplained $351.50."
-us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Left out the couple's $881/month of SSI and treated $630 of SSDI as total income. It then guessed a net income near $300 instead of using $1,511 − $209 − $145 = $1,157."
-us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Counted only the $630/month of SSDI and left out the $881/month of SSI. It also handled the medical expenses wrongly, comparing the $632 annual figure against the monthly $35 threshold, instead of applying Arizona's $145 standard medical deduction."
-us,scenario_079,snap,claude-opus-5.5,llm_error,state_local_rule,False,"Counted SSI and used the right standard deduction ($209) and maximum ($546), but deducted only the $26 actual medical excess. Arizona's $145 standard medical deduction lowers net income to $1,157, so the benefit is $198/month, not $163."
-us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Said the household receives SSI but then counted only the $630 of SSDI, leaving the $881/month of SSI out of gross income. It also used outdated parameters ($204 standard deduction, $535 maximum) and the $26 actual medical excess instead of Arizona's $145 standard medical deduction."
-us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Assumed large mortgage shelter costs from a loan balance, even though no mortgage payment or interest is listed, so shelter costs are $0. It also left the $881/month of SSI out of income; the correct net income is $1,157, giving a $198/month benefit."
-us,scenario_079,snap,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Used $630/month of SSDI as total income and left out the $881/month of SSI the couple receives. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, so net income came out at $395 instead of $1,157."
-us,scenario_079,snap,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Counted SSI but used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used the wrong standard deduction ($228, not $209) and maximum allotment ($549, not $546); the correct figures give net income of $1,157 and $198/month."
-us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"Counted $5,050 of vehicle equity above a $4,650 exclusion as a resource. The household is categorically eligible, and its vehicles aren't countable, so the only countable asset is the $1,045 bank balance, which passes. The benefit is $198/month."
-us,scenario_079,snap,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Counted SSI, but at $872/month instead of $881, and deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $202 standard deduction and $548 maximum instead of $209 and $546."
-us,scenario_079,snap,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Took net income from $630 of SSDI alone and left out the $881/month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, so net income was $400 instead of $1,157."
-us,scenario_079,snap,gemini-3-flash-preview,llm_error,state_local_rule,False,"Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also understated SSI ($805 vs $881) and used old parameters ($198 standard deduction, $535 maximum) instead of $209 and $546."
-us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"Concluded that having no rent or utilities means a zero benefit. SNAP doesn't require shelter costs: $1,511 − $209 − $145 = $1,157 net income gives $546 − $348 = $198/month."
-us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,other,False,"Claimed 30% of net income exceeds the maximum allotment and gave only a minimum benefit. Net income is $1,157/month, so the contribution is $348, well below the $546 maximum, and the benefit is $198/month."
-us,scenario_079,snap,gemini-3.5-flash,llm_error,state_local_rule,False,"Deducted only $312/year ($26/month) of excess medical costs instead of Arizona's $145/month standard medical deduction. It also used a $560/month maximum allotment ($6,720/12) instead of $546."
-us,scenario_079,snap,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"Said the household fails the asset and income rules, but it passes: assets are $1,045, and gross income is 86% of the poverty guideline. Net income of $1,157 gives a $198/month benefit."
-us,scenario_079,snap,gemini-3.6-flash,llm_error,state_local_rule,False,"Counted SSI, but its $178/month result implies it used only the actual medical excess. Arizona's $145 standard medical deduction brings net income to $1,157 and the benefit to $198/month."
-us,scenario_079,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $470/month result implies it counted only the $630 of SSDI and left out the $881/month of SSI. With SSI counted and the $145 Arizona standard medical deduction applied, net income is $1,157 and the benefit is $198/month."
-us,scenario_079,snap,gemini-3.8-flash,llm_error,state_local_rule,False,"Counted only the head's $432 of other medical expenses, dropping the OTC costs, for a $12 deduction. Arizona's $145 standard medical deduction applies instead. It also used SSI of $840/month and a $536 maximum instead of $881 and $546."
-us,scenario_079,snap,gemini-4-argon,llm_error,state_local_rule,False,"Its ~$157/month result implies it counted SSI but deducted only the actual medical excess. Arizona's $145 standard medical deduction makes net income $1,157 and the benefit $198/month."
-us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,"Gave no parseable SNAP value. The correct derivation gives $198/month, or $2,376 a year."
-us,scenario_079,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"Started from $7,560 of SSDI alone and left the couple's $10,572/year of SSI out of gross income. It also deducted $312/year of actual medical excess instead of Arizona's $145/month standard medical deduction."
-us,scenario_079,snap,gpt-5.4-mini,llm_error,other,False,"Concluded that with no dependents or shelter costs the benefit is zero. A two-person household with $1,157 net income qualifies for $546 − $348 = $198/month without any shelter costs."
-us,scenario_079,snap,gpt-5.4-nano,llm_error,other,False,"Said there was too little information and answered 0, even though income and deductions are fully specified. They give net income of $1,157 and a $198/month allotment."
-us,scenario_079,snap,gpt-5.5,llm_error,state_local_rule,False,"Counted SSI correctly but used a $204 standard deduction and the $26 actual medical excess instead of $209 and Arizona's $145 standard medical deduction. That gave net income of $1,281 instead of $1,157."
-us,scenario_079,snap,gpt-5.6-luna,llm_error,other,False,"Gave the full $546 maximum allotment and never subtracted the 30% expected contribution. With net income of $1,157, the contribution is $348, so the benefit is $198/month."
-us,scenario_079,snap,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $163/month result comes from net income of about $1,276, which uses the $26 actual medical excess. Arizona's $145 standard medical deduction makes net income $1,157 and the benefit $198/month."
-us,scenario_079,snap,gpt-5.6-terra,llm_error,state_local_rule,False,"Counted SSI but its $165/month result reflects only the actual excess medical deduction. Arizona's $145 standard medical deduction lowers net income to $1,157 and raises the benefit to $198/month."
-us,scenario_079,snap,gpt-6-astra,llm_error,state_local_rule,False,"Correctly used $1,511 income, the $209 standard deduction and the $546 maximum, but deducted $26 of actual medical excess instead of Arizona's $145 standard medical deduction. It also skipped rounding the 30% contribution up to a whole dollar."
-us,scenario_079,snap,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its $395 net income comes from $630 of SSDI alone, leaving out the $881/month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction."
-us,scenario_079,snap,gpt-6-sol,llm_error,state_local_rule,False,"Counted SSI and used the right standard deduction and maximum, but deducted only $26 of actual medical excess instead of Arizona's $145 standard medical deduction. That gives $163/month instead of $198."
-us,scenario_079,snap,gpt-6.1-sol,llm_error,state_local_rule,False,"Deducted $26 of actual medical excess instead of Arizona's $145 standard medical deduction, which left net income at $1,276 instead of $1,157. It also didn't round the 30% contribution up."
-us,scenario_079,snap,gpt-6.1-terra,llm_error,state_local_rule,False,"Its $163/month result reflects the $26 actual medical excess. Arizona's $145 standard medical deduction brings net income to $1,157 and the benefit to $198/month."
-us,scenario_079,snap,grok-4.3,llm_error,thresholds_rates,False,"Asserted that income exceeds SNAP limits. Gross income of $1,511 is 86% of the poverty guideline and net income of $1,157 passes the net test, so the household gets $198/month."
-us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"Treated $7,560 of SSDI as the only income and left out the couple's $881/month of SSI. It also deducted $312/year of actual medical excess instead of Arizona's $145/month standard medical deduction."
-us,scenario_079,snap,grok-4.6,llm_error,state_local_rule,False,"Counted SSI, but at about $876, and deducted only $26 of actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $549 maximum instead of $546."
-us,scenario_079,snap,grok-4.7,llm_error,state_local_rule,False,"Correctly computed $1,511 income, the $209 standard deduction and the $546 maximum, but used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also didn't round the contribution up."
-us,scenario_079,snap,grok-build-0.1,llm_error,state_local_rule,False,"Used 2025 figures: SSI of $840, a $204 standard deduction and a $536 maximum, instead of 2026's $881, $209 and $546. It also deducted $26 of actual medical excess instead of Arizona's $145 standard medical deduction."
-us,scenario_079,snap,inkling,llm_error,taxable_income_or_deductions,False,Built net income from $630 of SSDI alone and left out the $881/month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction.
-us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no parseable SNAP value. The correct derivation gives $198/month, or $2,376 a year."
-us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,Started from $630 of SSDI and left the $881/month of SSI out of gross income. It also deducted the $26 actual medical excess instead of Arizona's $145 standard medical deduction.
-us,scenario_079,snap,minimax-m3,llm_error,other,False,"Gave 0 because it wouldn't infer benefit receipt, but the prompt says to assume program take-up. The eligible household's benefit is $546 − $348 = $198/month."
-us,scenario_079,snap,muse-spark-1.3,llm_error,state_local_rule,False,"Counted SSI, but used a $204 standard deduction, a $536 maximum and the $26 actual medical excess instead of $209, $546 and Arizona's $145 standard medical deduction."
-us,scenario_079,snap,muse-wonder,llm_error,taxable_income_or_deductions,False,Treated $630 of SSDI as all of the household's income and left out the $881/month of SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction.
-us,scenario_079,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"Counted only the $630/month of SSDI and left out the $881/month of SSI, so net income came out at about $399 instead of $1,157. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction."
-us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Left out the $881/month of SSI and treated the $35 medical threshold as annual, giving a $697 deduction. Arizona's $145/month standard medical deduction applies instead. It also used a $598 maximum instead of $546."
-us,scenario_079,snap,qwen3.8-max,llm_error,state_local_rule,False,"Applied an SSI cash-out rule that shuts SSI recipients out of SNAP, a rule California used until 2019, but Arizona has no such rule. In Arizona, SSI receipt makes the couple categorically eligible, and they get $198/month."
+us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Counted only the $630 SSDI as gross income and left out the couple's $881/month SSI ($1,491 couple rate minus $610 countable SSDI), which is SNAP unearned income. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It then swapped its own $417.50/month result for an arbitrary $4,560."
+us,scenario_079,snap,claude-fable-5.1,llm_error,state_local_rule,False,"Got the $1,511 gross income and $209 standard deduction right, but deducted only the $26 actual medical excess. Arizona's $145 standard medical deduction applies to elderly/disabled households with more than $35 of medical costs. Correct net income is $1,157, giving a $348 contribution and $198/month, not $163."
+us,scenario_079,snap,claude-fable-5.2,llm_error,state_local_rule,False,"Used the $26 actual excess medical deduction instead of Arizona's $145 standard medical deduction for elderly/disabled households. That left net income at $1,276 instead of $1,157. It also did not round the 30% contribution up to a whole dollar, which produced $163.20/month instead of $198."
+us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"Disqualified the household on home and vehicle equity, but the home is an excluded resource and vehicles are not counted. Countable assets are only the $1,045 bank account, which passes, and the SSI household is categorically eligible. It also left out the $881/month SSI. The correct benefit is $198/month from $1,511 gross income and $354 of deductions."
+us,scenario_079,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"Claimed $630/month of SSDI exceeds the two-person income limit, but it is far below it. Even with the omitted $881 SSI, the $1,511 gross is 86% of the poverty guideline, and net income of $1,157 yields a $198/month benefit."
+us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,Left the couple's $881/month SSI out of gross income and counted only the $630 SSDI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It then replaced its own $420.50/month result with an unsupported $351.50.
+us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Treated SSDI as the only income and never counted the $881/month SSI the disabled couple receives. That understated gross income by more than half. The guessed net income of about $300 does not come from Arizona's $145 standard medical deduction applied to $1,511 of income."
+us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Counted only the $630 SSDI and left out the couple's $881/month SSI, which is countable unearned income. It also never computed the medical deduction: Arizona's $145 standard medical deduction applies, giving $1,157 of net income and a $198/month benefit."
+us,scenario_079,snap,claude-opus-5.5,llm_error,state_local_rule,False,"Included SSI correctly but deducted only the $26 actual excess over $35. Arizona's $145 standard medical deduction for elderly/disabled households lowers net income to $1,157 and raises the benefit to $198/month."
+us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Treated SSDI as the only income, although its own reasoning says both members receive SSI/SSDI. It never added the $881/month SSI to gross income. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, a $204 standard deduction instead of $209, and a $535 maximum allotment instead of $546."
+us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"Invented an excess shelter deduction from the mortgage balance even though no mortgage payment, interest, or utility cost was given, so shelter costs are $0. It also left the $881/month SSI out of gross income. The correct calculation uses only the $209 standard deduction and $145 Arizona standard medical deduction against $1,511 of income."
+us,scenario_079,snap,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"Counted only the $630 SSDI and left out the couple's $881/month SSI, which raises gross income to $1,511. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction."
+us,scenario_079,snap,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Used the $26 actual medical excess instead of Arizona's $145 standard medical deduction for elderly/disabled households. It also used a $228 standard deduction instead of $209, $876 of SSI instead of $881, and a $549 maximum allotment instead of $546. The correct net income of $1,157 gives $198/month."
+us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"Counted $5,050 of vehicle equity as a resource and failed the household on assets. Vehicles are not counted, so only the $1,045 bank balance counts, and the SSI-receiving household is categorically eligible. The household qualifies for $198/month."
+us,scenario_079,snap,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Applied the actual $26 medical excess ($732 minus a $420 annual floor) instead of Arizona's $145 monthly standard medical deduction. It also used a $202 standard deduction, $872 of SSI, and a $548 maximum allotment, which left net income at $1,274 instead of $1,157."
+us,scenario_079,snap,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Counted only the $630 SSDI and left out the couple's $881/month SSI, so gross income was $630 instead of $1,511. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction."
+us,scenario_079,snap,gemini-3-flash-preview,llm_error,state_local_rule,False,Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction for elderly/disabled households. It also understated SSI at $805 (correct: $881) and used a $198 standard deduction and $535 maximum allotment instead of $209 and $546.
+us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"Concluded that a household with no rent or utility costs gets $0 SNAP, but shelter costs are not required for a benefit. With $1,511 of income, the $209 standard deduction, and the $145 Arizona standard medical deduction, net income is $1,157 and the benefit is $546 − $348 = $198/month."
+us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,other,False,"Claimed 30% of net income exceeds the maximum allotment and gave the minimum benefit. Net income after the $209 standard and $145 Arizona standard medical deductions is $1,157, and 30% of that is $348, well below the $546 maximum. The benefit is $198/month, not the minimum."
+us,scenario_079,snap,gemini-3.5-flash,llm_error,state_local_rule,False,Used the $312/year ($26/month) actual medical excess instead of Arizona's $145/month standard medical deduction for elderly/disabled households. It also used a $200 standard deduction and a $560 maximum allotment instead of $209 and $546.
+us,scenario_079,snap,gemini-3.5-flash-lite,llm_error,asset_resource,False,"Asserted unspecified asset and income barriers, but countable assets are only $1,045 and gross income of $1,511 is 86% of the poverty guideline. The SSI household is categorically eligible and receives $198/month."
+us,scenario_079,snap,gemini-3.6-flash,llm_error,state_local_rule,False,"Included SSI, but its $178/month result implies the $26 actual medical excess rather than Arizona's $145 standard medical deduction. The correct net income of $1,157 gives a $348 contribution and $198/month."
+us,scenario_079,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $470/month benefit is consistent with counting only the $630 SSDI and leaving out the couple's $881/month SSI. Correct gross income is $1,511, which with $354 of deductions gives $1,157 of net income and $198/month."
+us,scenario_079,snap,gemini-3.8-flash,llm_error,state_local_rule,False,"Computed a $12 medical deduction from the $432 other medical expense alone, leaving out OTC costs and ignoring Arizona's $145/month standard medical deduction for elderly/disabled households. It also used $840 of SSI and a $536 maximum allotment instead of $881 and $546."
+us,scenario_079,snap,gemini-4-argon,llm_error,state_local_rule,False,"Its $157/month result reflects SSI plus SSDI income with only the actual medical excess over $35 deducted. Arizona's $145 standard medical deduction reduces net income to $1,157, which gives $198/month."
+us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,"Gave no SNAP value or explanation, so no answer could be scored against the $2,376 reference."
+us,scenario_079,snap,glm-5.3,llm_error,taxable_income_or_deductions,False,"Counted only the $7,560 of SSDI and left out the couple's $10,572/year ($881/month) of SSI. It also used the $312/year actual medical excess instead of Arizona's $145/month standard medical deduction."
+us,scenario_079,snap,gpt-5.4-mini,llm_error,other,False,"Concluded that no dependents or shelter costs means $0 SNAP, but a two-person disabled household with $1,157 of net income qualifies without either. The benefit is $546 − $348 = $198/month."
+us,scenario_079,snap,gpt-5.4-nano,llm_error,other,False,"Declined to compute SNAP and returned $0, even though the facts fully determine the benefit. Gross income is $1,511 (SSDI plus SSI), deductions are $354, net income is $1,157, and the benefit is $198/month."
+us,scenario_079,snap,gpt-5.5,llm_error,state_local_rule,False,"Included SSI but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $204 standard deduction instead of $209. That left net income at $1,281 instead of $1,157, giving $161/month instead of $198."
+us,scenario_079,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Awarded the full $546 maximum allotment as if net income were zero, ignoring $1,511/month of SSDI and SSI. After the $209 standard and $145 medical deductions, net income is $1,157, which brings in a $348 contribution."
+us,scenario_079,snap,gpt-5.6-sol,llm_error,state_local_rule,False,"Its $163/month result matches $1,511 of income less only the $26 actual medical excess. It missed Arizona's $145 standard medical deduction for elderly/disabled households, which lowers net income to $1,157 and gives $198/month."
+us,scenario_079,snap,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $165/month benefit reflects SSI plus SSDI income with only the actual medical excess over $35 deducted. Arizona's $145 standard medical deduction cuts net income to $1,157 and raises the benefit to $198/month."
+us,scenario_079,snap,gpt-6-astra,llm_error,state_local_rule,False,"Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction, and did not round the 30% contribution up. The correct figures are $1,157 net income, a $348 contribution, and $198/month."
+us,scenario_079,snap,gpt-6-luna,llm_error,taxable_income_or_deductions,False,Its $395 net income comes from counting only the $630 SSDI and leaving out the couple's $881/month SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction.
+us,scenario_079,snap,gpt-6-sol,llm_error,state_local_rule,False,Included SSI but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. That overstated net income by $119 and gave $163/month instead of $198.
+us,scenario_079,snap,gpt-6.1-sol,llm_error,state_local_rule,False,"Used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, which left net income at $1,276 instead of $1,157. It also did not round the 30% contribution up to $348."
+us,scenario_079,snap,gpt-6.1-terra,llm_error,state_local_rule,False,"Its $163/month result reflects SSI plus SSDI income with the actual $26 medical excess. It missed Arizona's $145 standard medical deduction for elderly/disabled households, which gives $1,157 of net income and $198/month."
+us,scenario_079,snap,grok-4.3,llm_error,thresholds_rates,False,"Claimed SSDI income exceeds the SNAP limits, but $1,511 of gross income is 86% of the poverty guideline and $1,157 of net income is under the net limit. The household receives $198/month."
+us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"Counted only the $7,560 of SSDI and left out the couple's $881/month SSI. It also used the $312/year actual medical excess instead of Arizona's $145/month standard medical deduction."
+us,scenario_079,snap,grok-4.6,llm_error,state_local_rule,False,"Included SSI but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $549 maximum allotment instead of $546, ending at $168/month instead of $198."
+us,scenario_079,snap,grok-4.7,llm_error,state_local_rule,False,"Applied the $26 actual excess on $732 of medical costs instead of Arizona's $145 standard medical deduction for elderly/disabled households. It also did not round the 30% contribution up, which gave $163.20/month instead of $198."
+us,scenario_079,snap,grok-build-0.1,llm_error,state_local_rule,False,"Deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used 2025 SSI rates ($840/month instead of $881), a $204 standard deduction, and a $536 maximum allotment instead of the 2026 figures of $209 and $546."
+us,scenario_079,snap,inkling,llm_error,taxable_income_or_deductions,False,"Counted only the $630 SSDI and left out the couple's $881/month SSI, so net income came out at $395 instead of $1,157. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction."
+us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no SNAP value or explanation, so no answer could be scored against the $2,376 reference."
+us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"Waived the gross income test correctly but counted only the $630 SSDI, leaving out the couple's $881/month SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction."
+us,scenario_079,snap,minimax-m3,llm_error,other,False,"Returned $0 on the grounds that benefit receipt cannot be inferred, but the prompt says to assume program take-up. SNAP is a computed output: $1,511 of income less $354 of deductions gives $1,157 of net income and $198/month."
+us,scenario_079,snap,muse-spark-1.3,llm_error,state_local_rule,False,Included SSI but deducted only the $26 actual medical excess instead of Arizona's $145 standard medical deduction. It also used a $204 standard deduction and a $536 maximum allotment instead of $209 and $546.
+us,scenario_079,snap,muse-wonder,llm_error,taxable_income_or_deductions,False,"Counted only the $630 SSDI and left out the couple's $881/month SSI, which brings gross income to $1,511. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction."
+us,scenario_079,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"Counted only the $630 SSDI and left out the couple's $881/month SSI. It also used the $26 actual medical excess instead of Arizona's $145 standard medical deduction, a $205 standard deduction instead of $209, and a $547 maximum allotment instead of $546."
+us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Counted only the $7,560 of SSDI and left out the $881/month SSI. It also applied the $35 medical threshold annually instead of monthly and used a $598 maximum allotment instead of $546. The correct calculation uses Arizona's $145/month standard medical deduction against $1,511 of monthly income."
+us,scenario_079,snap,qwen3.8-max,llm_error,state_local_rule,False,"Assumed SSI recipients get no separate SNAP benefit, but that SSI cash-out applies only in California. In Arizona, SSI receipt makes the household categorically eligible, and the household receives $198/month."
us,scenario_079,spouse_medicaid_eligible,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"The model asserted that the spouse's ""income and circumstances"" fail Arizona's adult Medicaid rules, inverting the test: the household's MAGI is 0.35 × FPL, far below Arizona's 138% FPL expansion-adult limit, so low income qualifies rather than disqualifies. It also skipped the pathway the engine used — the SSI-recipient category, which grants automatic Medicaid in Arizona (a 1634 state) to the blind/disabled spouse whose $0 countable income and $1,045 in resources (under the $2,000 individual limit) produce $5,286 of SSI — and treated an income screen as the sole route to adult coverage."
us,scenario_079,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model defaulted to 0 on the ground that ""no additional Medicaid-eligibility inputs"" beyond disability and IHS coverage were supplied, treating Medicaid eligibility as requiring an explicit enrollment or income input rather than deriving it from the facts given. Blindness plus disability plus $0 earned and unearned income and $1,045 in countable resources produce SSI of $5,286, and SSI receipt in Arizona triggers the SSI-recipient category's automatic Medicaid; separately, 0.35 × FPL MAGI clears the 138% FPL expansion-adult test, and IHS coverage at interview is not minimum essential coverage that blocks eligibility."
us,scenario_079,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated the spouse's disabled flag as enough for Medicare under 65, citing 'SSDI-linked disability eligibility'. However, the spouse gets no Social Security disability income; the $7,560 SSDI is the head's. No months of SSDI receipt are listed, so the 24-month SSDI entitlement requirement is not met and the spouse is not Medicare eligible."
@@ -7192,57 +7268,58 @@ us,scenario_080,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_erro
us,scenario_080,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable output for state_refundable_credits, violating the required submission contract."
us,scenario_080,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model misclassified Pennsylvania Schedule SP tax forgiveness as a refundable credit that pays out the $28.86 liability. Schedule SP is nonrefundable and only reduces state income tax, so it contributes $0 to state_refundable_credits."
us,scenario_080,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Set Massachusetts income tax in SALT to zero and itemized $22,543, leaving taxable income near $151,550. The excluded frozen reference itemizes $31,027.41 using an unlisted $8,484.41 withholding estimate and has taxable income of $143,059.47. It then raised its own bracket result of $28,759 to $29,385 with no basis."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Used about $8,230 of MA income tax in SALT instead of PolicyEngine's $8,484.41 estimate of MA tax withheld (5% of AGI above $4,400). It also added the $6 of REIT income to AGI. Together these put taxable income at about $143,320, compared with $143,059.47 in the excluded frozen reference, which uses unlisted estimated withholding."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Wrongly concluded that $16,900 of mortgage interest falls below the standard deduction and never itemized. The excluded frozen reference itemizes $31,027.41 using an unlisted withholding estimate. It also took the non-deductible $1,782 of employee business expenses above the line and used the 2024 $14,600 deduction and 2024 brackets. Finally it replaced its own $30,957 with an arbitrary $29,825."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Gave a rounded $33,000 with no derivation. That exceeds even the $30,514.85 tax under the standard deduction. The excluded frozen reference itemizes $31,027.41, including unlisted estimated withholding, and has taxable income of $143,059.47."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at the old $10,000 limit. The 2026 OBBBA cap is $40,400. The excluded frozen reference deducts $14,127.41 of SALT and itemizes $31,027.41 using an unlisted withholding estimate. It then discarded its own $27,887 bracket result and inflated it to $31,369."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Applied the expired $10,000 SALT cap instead of the 2026 $40,400 cap and itemized $26,900. The excluded frozen reference itemizes $31,027.41 using an unlisted withholding estimate. It also used 2025 bracket thresholds. It then replaced its own $28,172 with an unsupported $31,379."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized mortgage interest and real estate tax ($22,543), giving taxable income of about $151,544. The excluded frozen reference adds an unlisted $8,484.41 estimate of MA withholding to SALT. It then inflated its own $28,968 bracket result to $31,650 without justification."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Put the computed MA liability of $8,238.70 into SALT instead of PolicyEngine's withholding estimate of 5% × (AGI − $4,400) = $8,484.41. It also counted the $6 of REIT income in AGI. Taxable income was $251.63 above the excluded frozen withholding-based reference ($143,311.10 vs $143,059.47). Its $26,992.66 is $0.02 above the recorded corrected-liability reading of $26,992.64."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Limited SALT to the $5,643 of real estate tax and itemized $22,543. The excluded frozen reference itemizes $31,027.41 using an unlisted $8,484.41 withholding estimate. It also used 2025 bracket thresholds instead of the 2026 thresholds (12,400/50,400/105,700), giving taxable income of $151,551 and tax of $29,219."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at $10,000 instead of the 2026 $40,400 cap, putting itemized deductions $4,127 below the excluded frozen reference, which uses an unlisted withholding estimate. Even its own $147,194 of taxable income gives about $27,925 on the 2026 single brackets, so its $33,700 is also a bracket arithmetic error."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Estimated MA income tax at about $8,284 instead of PolicyEngine's $8,484.41 withholding estimate (5% of AGI above $4,400). It also included the $6 of REIT income in AGI. Taxable income of about $143,266 is about $207 above the excluded frozen reference, which uses unlisted estimated withholding."
-us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at the pre-2025 $10,000 limit and itemized $26,900. The excluded frozen reference itemizes $31,027.41 using an unlisted withholding estimate. Its taxable income of $147,194 compares with $143,059.47 in that reference."
-us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"Applied pre-TCJA law (a $5,400 personal exemption and 10/15/25/28% brackets), even though OBBBA made the TCJA rate structure permanent for 2026. It also subtracted the $7,389 of employer insurance premiums from the listed wages and left MA income tax out of SALT."
-us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Used a $5,312 personal exemption and pre-TCJA single brackets, which OBBBA eliminated for 2026. Its $12,754.55 of SALT compares with $14,127.41 in the excluded frozen reference, which uses an unlisted withholding estimate. It submitted $27,594.14 even though its own explanation computes $29,794.41."
-us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Deducted $7,539.70 of MA income tax and added the $6 of REIT income to AGI. Its taxable income of $144,010.10 is about $950 above the excluded frozen reference's $143,059.47, whose SALT uses an unlisted withholding estimate."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA had expired, subtracting a $5,050 personal exemption and applying 15/25/28% brackets. Under OBBBA's permanent extension, 2026 has no personal exemption and uses 10/12/22/24% brackets, which give $26,932.27 on $143,059.47 of taxable income."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Took a $15,000 standard deduction even though mortgage interest and real estate tax alone total $22,543. The excluded frozen reference itemizes $31,027.41 using an unlisted withholding estimate. Its $31,317 also exceeds the $30,514.85 that the actual 2026 standard deduction would produce."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-expired brackets and a personal exemption, which no longer exist in 2026 after OBBBA. It also cut AGI to $166,705 by subtracting the $7,389 of employer insurance premiums from wages, which are already the taxable gross."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Applied TCJA-expired law with a $5,300 personal exemption and pre-TCJA brackets, and subtracted the $7,389 of employer insurance premiums from the listed wages. It also left the $8,484.41 of estimated MA income tax out of SALT, itemizing only $22,543."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Reported taxable income of $169,008, compared with $143,059.47 in the excluded frozen reference, whose $31,027.41 of itemized deductions includes unlisted estimated withholding. Its $24,197 does not follow from 2026 brackets on its own base either, since those give about $33,160."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Subtracted the $7,389 of employer insurance premiums from the listed wages, which PolicyEngine treats as the taxable wage figure. It also left the $8,484.41 of estimated MA income tax out of SALT, itemizing only $22,543. Its $27,858 also overstates the 2026 bracket tax on its own $144,151 of taxable income, which is about $27,194."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Gave no breakdown. Its $24,907 implies taxable income of about $134,600 on the 2026 brackets, roughly $8,400 below the excluded frozen reference's $143,059.47, whose deductions include unlisted estimated withholding."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at $10,000 even though the 2026 cap is $40,400 and itemized $26,900. The excluded frozen reference itemizes $31,027.41 using an unlisted withholding estimate. Its submitted $28,050 is $1,117.73 above that frozen value."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"Used the computed MA liability of $8,238 in SALT instead of PolicyEngine's withholding estimate of 5% × (AGI − $4,400) = $8,484.41. Itemized deductions of $30,781 and taxable income of $143,305.80 compare with $31,027.41 and $143,059.47 in the excluded frozen withholding-based reference. Its $26,991.39 is $0.08 above the recorded liability-based reading of $26,991.31."
-us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Treated 2026 as a TCJA-sunset year, with a $5,150 personal exemption, uncapped SALT under pre-TCJA rules, a 10% medical floor and 10/15/25/28% brackets. OBBBA kept the TCJA rate schedule and eliminated personal exemptions."
-us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Used $8,388.70 of MA income tax in SALT and counted the $6 of REIT income in AGI, giving taxable income of $143,161.10. The excluded frozen reference uses an unlisted $8,484.41 withholding estimate and has taxable income of $143,059.47. It then mis-added its own bracket pieces: $17,966 + $8,990.66 = $26,956.66, not $26,916.66."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"Capped SALT at $10,000 instead of the 2026 $40,400 cap. Its $22,450 is also about $5,500 below what the 2026 brackets give even on that capped base (about $27,925), so it also misapplied the rate schedule."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Reported $41,495, more than the 2026 bracket tax on the entire $174,088 AGI with no deductions (about $34,380). It therefore misapplied the rate schedule. The excluded frozen reference's $31,027.41 of itemized deductions includes an unlisted withholding estimate."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Got itemized deductions roughly right (about $30,933) but also subtracted a personal exemption and used the pre-TCJA rate schedule, both repealed for 2026 under OBBBA. Taxable income of $143,059 on the 10/12/22/24% brackets gives $26,932.27."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Reduced the listed wages by the $7,389 of employer insurance premiums, which PolicyEngine does not subtract from the stated gross wages. Its $24,920 implies taxable income of about $134,700, roughly $8,400 below $143,059.47."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its federal explanation gives no state-tax figure, and its answer is consistent with using its $8,238.70 Massachusetts liability estimate in SALT. Its $26,992.59 is $0.05 below the recorded corrected-liability reading of $26,992.64. The excluded frozen reference uses an unlisted $8,484.41 withholding estimate."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Understated the Massachusetts income tax part of SALT relative to PolicyEngine's $8,484.41 withholding estimate (5% of AGI above $4,400). Its $26,966.21 implies taxable income of about $143,201, which is $141 above $143,059.47."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized $30,773.10, using about $8,230 of MA income tax instead of PolicyEngine's $8,484.41 (5% of AGI above $4,400). Its $26,993.29 is about $61 above the excluded frozen withholding-based reference and $0.65 above the recorded corrected-liability reading of $26,992.64."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of about $143,260 is about $200 above the excluded frozen reference's $143,059.47, whose SALT uses an unlisted $8,484.41 withholding estimate. Its $26,980 is $11.31 below the recorded liability-based reading of $26,991.31."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Deducted $8,330.10 of MA income tax instead of PolicyEngine's $8,484.41 estimate of MA tax withheld (5% × ($174,088.12 − $4,400)). Its $26,969.29 is $37.02 above the excluded frozen withholding-based reference and $22.02 below the recorded liability-based reading of $26,991.31."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized $30,773.10, using about $8,230 of MA income tax instead of the $8,484.41 withholding estimate that PolicyEngine puts in SALT. Its $26,993.29 is about $61 above the excluded frozen withholding-based reference and $0.65 above the recorded corrected-liability reading of $26,992.64."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Understated the state income tax part of SALT relative to PolicyEngine's $8,484.41 estimate (5% of AGI above $4,400). Its $26,980.26 implies taxable income of about $143,259, which is $200 above $143,059.47."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of about $151,500 is consistent with deducting only mortgage interest and property tax. The excluded frozen reference itemizes $31,027.41 using an unlisted $8,484.41 withholding estimate and has taxable income of $143,059.47."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed the TCJA sunset in 2026, subtracting a personal exemption of about $5,447 and applying restored 2017-style 10/15/25/28% brackets. OBBBA made the 10/12/22/24% schedule permanent and eliminated personal exemptions."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Applied a post-TCJA-sunset return with a $5,300 personal exemption and restored 10/15/25/28% brackets. Under OBBBA, 2026 keeps the TCJA brackets with no exemption, so $143,059.47 of taxable income gives $26,932.27."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Treated 2026 as a TCJA-sunset year, deducting a $5,350 personal exemption and applying 10/15/25/28% brackets. OBBBA eliminated the exemption and kept the 10/12/22/24% schedule."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Applied reverted pre-TCJA brackets that OBBBA superseded for 2026. It also itemized only $22,543, leaving the $8,484.41 of estimated MA income tax out of SALT."
-us,scenario_081,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"Used about $8,335 of MA income tax in SALT and included the $6 of REIT income in AGI, giving taxable income of about $143,215. The excluded frozen reference uses an unlisted $8,484.41 withholding estimate. It then reported $27,056, even though the 2026 brackets on its own base give about $26,970."
-us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,Returned no value and no explanation for federal_income_tax_before_refundable_credits.
-us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized $22,543 (mortgage interest plus real estate tax), giving taxable income of $151,543.80. The excluded frozen reference adds an unlisted $8,484.41 withholding estimate and has taxable income of $143,059.47."
-us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"Reported $0 with no computation. The excluded frozen reference uses unlisted estimated withholding and computes $26,932.27 on taxable income of $143,059.47."
-us,scenario_081,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"Itemized $22,543, giving taxable income of about $151,551. The excluded frozen reference adds an unlisted $8,484.41 withholding estimate and has taxable income of $143,059.47."
-us,scenario_081,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"Used about $8,384 of MA income tax in SALT instead of PolicyEngine's $8,484.41 withholding estimate. It then reported $27,047, even though the 2026 brackets on its own $143,167 of taxable income give about $26,958."
-us,scenario_081,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Used $8,538 of MA income tax in SALT and included the $6 of REIT income in AGI, giving taxable income of about $143,012. Its $26,921 is $11.27 below the excluded frozen withholding-based reference and $70.31 below the recorded liability-based reading of $26,991.31."
-us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"Wrongly claimed $22,543 of itemized deductions was below a $15,000 standard deduction and took the standard deduction. The excluded frozen reference itemizes $31,027.41 using an unlisted withholding estimate. It then submitted $39,827.88, contradicting its own computed $31,281.32."
-us,scenario_081,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"Wrongly concluded that $22,543 of itemized deductions was lower than the standard deduction and took the standard deduction. It also left out the $8,484.41 of estimated MA income tax that makes itemized deductions $31,027.41."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated Massachusetts income tax as $0 in SALT. That left out the $8,484.41 withholding estimate (5% of AGI above $4,400) and cut itemized deductions to $22,543. It then raised its own $28,759 bracket result to $29,385 with no stated reason."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It entered about $8,230 of Massachusetts tax liability in SALT instead of the $8,484.41 withholding estimate (5% × ($174,088.12 − $4,400)). It also added the $6 of REIT income to AGI. Together these put taxable income about $260 too high and tax about $63 too high."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the 5% Massachusetts withholding estimate to gross wages of $175,002 ($8,530.10) instead of federal AGI of $174,088.12 ($8,484.41). That overstated SALT by $45.69, so taxable income came out at $143,013.70 instead of $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted the $1,782 of unreimbursed employee business expenses above the line, which is not allowed. It used the 2024 $14,600 standard deduction and 2024 brackets, and skipped itemizing even though mortgage interest plus SALT ($31,027) exceeds the deduction. Its final $29,825 does not match its own $30,956.92 computation."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It gave a rounded $33,000 with no derivation. Itemizing $31,027 (mortgage interest plus SALT including the $8,484.41 Massachusetts withholding estimate) leaves $143,059.47 of taxable income and $26,932.27 of tax. Its number fits ignoring the itemized deductions and overtaxing the income."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the pre-OBBBA $10,000 limit, but the 2026 cap is $40,400, so the full $14,127.41 is deductible. It then raised its own $27,887 bracket result to $31,369 with no basis."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the old $10,000 SALT cap instead of the 2026 $40,400 cap, and used 2025 bracket thresholds. It then submitted $31,379 even though its own computation gave about $28,172."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only real estate tax in SALT and left out the $8,484.41 Massachusetts withholding estimate, so taxable income was overstated at about $151,544. It then raised its own $28,968 result to $31,650."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It entered Massachusetts tax liability ($8,238.70) in SALT instead of the $8,484.41 withholding estimate. It also counted dividends as $116 by adding the REIT income, which put AGI at $174,094. Taxable income came out at $143,311.10 instead of $143,059.47, about $60 of extra tax."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $5,643 of real estate tax in SALT and left out the $8,484.41 Massachusetts income tax withholding estimate. It also used 2025 bracket thresholds instead of the 2026 ones ($12,400/$50,400/$105,700), so taxable income of $151,551 was overtaxed."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of the 2026 $40,400 cap. It then stated a tax of $33,700 on $147,194 of taxable income, but the 2026 brackets give about $27,925 on that amount."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated Massachusetts income tax at about $8,284 instead of the $8,484.41 withholding estimate (5% × (AGI − $4,400)), and included the $6 of REIT income in AGI. Taxable income came out at about $143,266 instead of $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at the pre-OBBBA $10,000 limit, but the 2026 cap is $40,400, so the full $14,127.41 of real estate tax plus Massachusetts withholding is deductible. This put taxable income at $147,194 instead of $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA law with a $5,400 personal exemption and 15/25/28% brackets, but the TCJA brackets continue in 2026 with no personal exemption. It also subtracted the $7,389 of ESI premiums from wages that were already reported, and left Massachusetts income tax out of SALT."
+us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA rules (a $5,312 personal exemption and pre-TCJA brackets), but the 2026 TCJA-extended brackets apply with no exemption. Its submitted $27,594.14 also contradicts the $29,794.41 in its own reasoning."
+us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used $7,539.70 of state income tax in SALT instead of the $8,484.41 Massachusetts withholding estimate (5% × ($174,088.12 − $4,400)), and included the $6 of REIT income in AGI. Taxable income came out at $144,010.10 instead of $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied a $5,050 personal exemption and the pre-TCJA 15/25/28% brackets, but 2026 law keeps the TCJA rate schedule (10/12/22/24%) with no personal exemption."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction instead of itemizing $31,027.41 (mortgage interest $16,900 plus SALT of $14,127.41, including the Massachusetts withholding estimate). That overstated taxable income by about $16,000."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used TCJA-expired brackets with a personal exemption, but TCJA rates continue in 2026. It also cut AGI to $166,705 by subtracting ESI premiums from wages that were already reported net."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA had expired and applied a $5,300 personal exemption and pre-TCJA brackets. It subtracted the $7,389 of ESI premiums from the reported wages, and left the $8,484.41 Massachusetts withholding estimate out of SALT."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It reported taxable income of $169,008, which subtracts only about $5,000 instead of $31,028.65 in itemized and QBI deductions. Its $24,197 tax figure also does not follow from that income on the 2026 brackets."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted the $7,389 of ESI premiums from wages that were already reported, and left the $8,484.41 Massachusetts withholding estimate out of SALT, itemizing only $22,543. Its $27,858 also does not match the 2026 brackets on its own taxable income of $144,151."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. $24,907 implies taxable income of about $134,600, roughly $8,400 below the correct $143,059.47. That fits excluding the $7,389 of ESI premiums from already-reported wages on top of itemizing."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It capped SALT at $10,000, but the 2026 OBBBA cap is $40,400, so the full $14,127.41 is deductible. This raised taxable income to $147,187 instead of $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"It entered Massachusetts tax liability ($8,238) in SALT instead of the $8,484.41 withholding estimate (5% × ($174,088.12 − $4,400)). That understated itemized deductions by about $246 and overstated tax by about $59."
+us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as post-TCJA-sunset, with a $5,150 personal exemption and pre-TCJA 15/25/28% brackets, but the 2026 schedule is 10/12/22/24% with no personal exemption."
+us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used $8,388.70 of Massachusetts income tax in SALT instead of the $8,484.41 withholding estimate, and included the $6 of REIT income in AGI, which gave taxable income of $143,161.10. It also mis-added its brackets: $17,966 plus 24% of $37,461.10 is $26,956.66, not $26,916.66."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the old $10,000 SALT cap instead of the 2026 $40,400 cap. Its $22,450 is also far below the 2026 bracket tax on any taxable income consistent with its own deductions."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no computation, and $41,495 grossly overstates the tax. Itemizing $31,027 (mortgage interest plus SALT including the Massachusetts withholding estimate) leaves $143,059.47 of taxable income and $26,932.27 of tax on the 2026 brackets."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It added an estimated 2026 personal exemption and used a pre-TCJA rate schedule, but the 2026 schedule has no personal exemption and uses 10/12/22/24% brackets."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It reduced wages by the $7,389 ESI premium, even though the reported wages are already the taxable amount. That double exclusion pushed taxable income about $7,400 below $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted Massachusetts tax liability (about $8,238) in SALT instead of the $8,484.41 withholding estimate (5% of AGI above $4,400). That understated itemized deductions by about $250 and overstated tax by about $60."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $26,966.21 implies taxable income of about $143,201. That means it used about $8,343 of Massachusetts income tax in SALT instead of the $8,484.41 withholding estimate."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It used $8,230.10 of Massachusetts income tax in SALT instead of the $8,484.41 withholding estimate (5% × ($174,088.12 − $4,400)). Itemized deductions came out at $30,773.10 instead of $31,027.41, and taxable income at $143,313.70."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It reached about $143,260 of taxable income by understating the Massachusetts income tax component of SALT by about $200, below the $8,484.41 withholding estimate (5% of AGI above $4,400)."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted $8,330.10 of state income tax instead of the $8,484.41 Massachusetts withholding estimate (5% × ($174,088.12 − $4,400)). Taxable income came out at $143,213.70 instead of $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It used $8,230.10 of Massachusetts income tax in SALT instead of the $8,484.41 withholding estimate. Itemized deductions came out at $30,773.10 instead of $31,027.41, and tax about $61 too high."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $26,980.26 implies taxable income of about $143,259. That means it put about $200 less state income tax in SALT than the $8,484.41 Massachusetts withholding estimate."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of about $151,500 shows that it itemized only mortgage interest and real estate tax and left the $8,484.41 Massachusetts withholding estimate out of SALT."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA sunset and applied a $5,447 personal exemption and restored 15/25/28% brackets, but 2026 keeps the 10/12/22/24% schedule with no personal exemption."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It applied a restored $5,300 personal exemption and 10/15/25/28% brackets on the assumption of a TCJA sunset, but the 2026 rate schedule is 10/12/22/24% with no exemption."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It treated 2026 as post-TCJA-sunset and applied a $5,350 personal exemption with 15/25/28% brackets, but 2026 law keeps the TCJA schedule with no personal exemption."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA 15/25/28% brackets, but the 2026 schedule is 10/12/22/24%. It also left the $8,484.41 Massachusetts withholding estimate out of SALT, which put taxable income at $151,551."
+us,scenario_081,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It used about $8,335 of state tax in SALT instead of the $8,484.41 Massachusetts withholding estimate, and included the $6 of REIT income in AGI. It then misapplied the brackets: $143,215 of taxable income gives about $26,970, not $27,056."
+us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no value and no explanation for federal_income_tax_before_refundable_credits.
+us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only mortgage interest and real estate tax ($22,543) and left the $8,484.41 Massachusetts withholding estimate out of SALT. That raised taxable income to $151,543.80."
+us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It submitted $0 with no derivation, which ignores the tax on $174,088 of AGI. Taxable income after the $31,027 of itemized deductions and the QBI deduction is $143,059.47, which is taxed at $26,932.27."
+us,scenario_081,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized only $22,543 and left the $8,484.41 Massachusetts withholding estimate out of SALT, which put taxable income at about $151,551. It also double-counted the REIT income in dividends ($116)."
+us,scenario_081,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It used about $8,384 of state tax in SALT instead of the $8,484.41 withholding estimate, and included the $6 of REIT income in AGI. It then overshot the brackets: $143,167 of taxable income gives about $26,958, not $27,047."
+us,scenario_081,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated the Massachusetts withholding estimate at $8,538 instead of 5% × ($174,088.12 − $4,400) = $8,484.41. That pushed taxable income down to $143,012 instead of $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $15,000 standard deduction after wrongly claiming that $22,543 of itemized deductions was smaller, and it left Massachusetts income tax out of SALT. Its submitted $39,827.88 also contradicts its own $31,281.32 bracket result."
+us,scenario_081,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It chose the standard deduction after wrongly concluding that itemized deductions were smaller. It never added the $8,484.41 Massachusetts withholding estimate to SALT; correct itemized deductions total $31,027.41, which beats the $16,100 standard deduction."
us,scenario_081,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_081,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It correctly worked out SS of $10,850.12, Medicare of $2,537.53 and a PFML contribution of $805.01 at 0.46%. It then dropped the PFML amount and submitted $13,268.70, which is $118.95 below even its own FICA-only sum of $13,387.65. The submitted value matches none of its own steps. The correct total, with PFML included, is $14,192.66."
us,scenario_081,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It computed FICA correctly at $13,387.65 but left out the Massachusetts PFML employee contribution (0.46% × $175,002 = $805.01). That fits the reading that the PFML employee share isn't a 'mandatory' employee tax, since employers can choose to pay it. The reference reads it as mandatory, assuming the maximum employee withholding."
@@ -7352,56 +7429,57 @@ us,scenario_082,child1_medicaid_eligible,minimax-m3,llm_error,health_coverage,Fa
us,scenario_082,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_082,child1_wic_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that the household met the WIC income requirement. It failed to compare approximately $90,132 of annual household income with the much lower 185%-of-poverty limit for a two-person household."
us,scenario_082,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the child's age under five as sufficient for WIC eligibility. Age establishes categorical eligibility only; the household must also pass the 185%-of-poverty income test, which it fails at approximately $90,132 of annual income."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own working ended near $10,353, but it submitted $14,748, which matches no step it showed. The working also left out the $68 SALT refund and the $700 non-itemizer charitable deduction, took the $8 IRA deduction even though the active-participant phase-out removes it, and used a $2,000 CTC instead of the 2026 amount of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its deductions ($24,150 standard plus $700 charitable) were correct, but its $10,444.60 ordinary tax on $79,380 is $80 too high. That implies a 12%/22% breakpoint near $66,650 instead of the 2026 HoH threshold of $67,450. It also took the $8 traditional IRA deduction, which is $0 because the head is covered by a 401(k) and AGI is above the HoH phase-out range."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It filed as single with a $14,600 standard deduction instead of head of household with the $24,150 deduction plus the $700 charitable deduction. It also let the capital loss offset income beyond the $3,000 cap. Its submitted $23,437 does not follow from its own pre-credit estimate of about $8,506."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave no bracket or deduction working and mentioned only the CDCC, never subtracting the $2,200 nonrefundable CTC. The correct chain is $92,811 taxable income and $12,379.77 gross tax, less $600 CDCC and $2,200 CTC, which gives $9,579.77. Its $14,500 overstates the tax by about $4,900."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $23,500 HoH standard deduction instead of $24,150, left out the $700 non-itemizer charitable deduction and the $68 SALT refund, and used pre-2026 HoH brackets (10% to $17,000, 12% to $64,850) instead of $17,700 and $67,450. Together these raised taxable income to $94,094 and ordinary tax to $10,922."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $23,000 HoH standard deduction, left out the $700 charitable deduction, took the $8 IRA deduction, used pre-2026 brackets (12% top at $64,850), and applied a $2,000 CTC instead of $2,200. It then raised its own $10,460 result to $11,150 with an unexplained adjustment."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It overstated the HoH standard deduction at $24,500 (the 2026 amount is $24,150) and left out the $700 non-itemizer charitable deduction. It also used outdated brackets (10% to $17,000, 12% to $64,850) instead of the 2026 HoH thresholds of $17,700 and $67,450, so its ordinary tax came to $10,716 instead of $10,366.17."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It deducted the $8 traditional IRA contribution and said outright that it assumed no active-participant phase-out. The head is a 401(k) participant with AGI well above the 2026 HoH phase-out range, so the IRA deduction is $0 and AGI is $117,661.12, not $117,654. Taxing those extra dollars at 22% accounts for its $1.57 shortfall."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It left the $13,424 of qualified dividends out of AGI but still subtracted them from taxable income to get ordinary income. It also added the $2,289 short-term gain on its own even though that gain is already netted into the capital loss. The result was an AGI of $106,519 instead of $117,661. On top of that it used a $22,500 standard deduction instead of $24,150 plus $700, brackets estimated by inflating 2024 thresholds, and a $2,000 CTC instead of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Its own working reached $10,739, but it submitted $13,920 with no supporting step. The working used a $22,500 HoH standard deduction instead of $24,150, left out the $700 charitable deduction and the $68 SALT refund, applied 2024 brackets (12% top at $63,100 instead of $67,450), and used a $2,000 CTC instead of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got AGI ($117,662), the $24,150 standard deduction, the brackets, and both credits right. It missed the 2026 charitable deduction for non-itemizers (OBBBA, up to $1,000 for HoH), which takes the $700 cash donation off taxable income. Its taxable income stayed at $93,512 instead of $92,811, adding $154 of tax at 22%."
-us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the full $33,000 long-term loss and then another $3,000, instead of netting to a single $3,000 capital-loss deduction. It used a nonexistent $46,000 HoH standard deduction, computed the CDCC as 35% of $5,000 instead of 20% of $3,000 ($600), and applied only $300 of CTC instead of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: $9,680 of personal exemptions, 10/15/25% brackets, and a $1,000 CTC. OBBBA made the TCJA structure permanent, so 2026 has no personal exemptions, a $24,150 HoH standard deduction plus the $700 charitable deduction, 10/12/22% brackets, and a $2,200 CTC."
-us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: $11,000 of personal exemptions, itemized deductions, pre-TCJA brackets, and a $1,000 CTC that phases out completely above the old $75,000 HoH threshold. Under OBBBA, 2026 uses the $24,150 standard deduction plus $700 charitable, 10/12/22% brackets, and a full $2,200 CTC with a $200,000 phase-out threshold."
-us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It took the $8 IRA deduction, used a $23,100 HoH standard deduction instead of $24,150, and left out the $700 charitable deduction. It computed $11,059.60 of tax on $81,130 of ordinary income, while 2026 HoH brackets give $10,749.60 on that amount. It also claimed a $2,300 CTC instead of the 2026 amount of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied TCJA-sunset law: $10,000 of personal exemptions, uncapped itemized SALT of $18,564 (double-counting the real estate tax), and a $1,000 CTC. Under OBBBA there are no personal exemptions, the $24,150 standard deduction plus $700 charitable deduction applies, and the CTC is $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its AGI of $100,571 drops about $17,090 of the dividend and investment income included in the correct $117,661. It subtracted only a $2,000 CTC and no $600 CDCC, while the correct credits total $2,800 ($2,200 CTC plus $600 CDCC) against $12,379.77 of gross tax."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 employer-sponsored insurance premiums from wages, giving AGI of $96,446 ($117,654 minus $21,208). Wages are reduced only by the 401(k) deferral, to $100,013.62. It also applied sunset-law personal exemptions ($10,400) and itemized deductions, and left out the $2,200 nonrefundable CTC entirely."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It excluded the $21,208 ESI premiums from wages, giving AGI of $96,454 instead of $117,661. It also applied TCJA-sunset rules with personal exemptions and a $1,000 CTC, instead of OBBBA's no exemptions, $24,150 standard deduction plus $700 charitable, and $2,200 CTC."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It showed no computation. The correct chain gives $12,379.77 of tax before credits and $9,579.77 after the $600 CDCC and $2,200 CTC. Its $12,790.25 is above even the pre-credit tax, which fits never subtracting the $2,800 of nonrefundable credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It applied TCJA-sunset personal exemptions, which do not exist in 2026 under OBBBA, and listed only the CDCC as a credit. The exemptions pushed taxable income below the correct $92,811, which is built from the $24,150 standard deduction plus the $700 charitable deduction. Its $7,380 is about $2,200 below the correct result of $12,379.77 gross tax minus $600 CDCC and $2,200 CTC."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It showed no computation. Its $3,384 is roughly $6,200 below the correct $9,579.77, which fits netting the full $30,711 capital loss against income instead of capping it at $3,000. With the full loss, all the qualified dividends fall into the 0% bracket and taxable income drops by about $27,700."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $2,000 CTC instead of the 2026 amount of $2,200 and left out the $700 non-itemizer charitable deduction. The remaining overstatement of about $320 comes from using pre-2026 HoH bracket thresholds instead of $17,700 and $67,450."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"It explicitly assumed the TCJA expired, so it used two personal exemptions, a roughly $12,000 HoH standard deduction, itemized SALT of $10,000, and a $1,000 CTC. OBBBA made TCJA permanent, so 2026 has no exemptions, a $24,150 HoH standard deduction plus $700 charitable, and a $2,200 CTC. It also took the $8 IRA deduction, which the active-participant phase-out removes."
-us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used an estimated $23,063 HoH standard deduction instead of $24,150, left out the $700 charitable deduction and the $68 SALT refund, and used estimated brackets (10% to $16,913, 12% to $64,575) instead of the 2026 HoH thresholds of $17,700 and $67,450. It also used a $2,000 CTC instead of $2,200."
-us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It counted the $21,208 ESI premiums as itemizable medical expenses. It then said its $23,266 itemized total beat the HoH standard deduction, although $24,150 is larger and the $700 non-itemizer charitable deduction comes on top of it. Its $11,046 tax on $80,972 of ordinary income also reflects pre-2026 brackets; 2026 HoH brackets give $10,714.84."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave no numeric working. The correct chain is $92,811 taxable income and $12,379.77 gross tax, minus $600 CDCC and $2,200 CTC, which gives $9,579.77. Its $12,719 is above the pre-credit tax, which fits failing to net the $2,800 of nonrefundable credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It chose to itemize using real estate taxes, medical/OTC costs, and charity. Medical costs of $1,965 fall below the 7.5%-of-AGI floor, so itemized deductions come to far less than the $24,150 HoH standard deduction plus the $700 non-itemizer charitable deduction. Its $17,303 is about $7,700 above the correct result from $92,811 of taxable income and $2,800 of credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It itemized $24,415 instead of taking the $24,150 standard deduction plus the $700 non-itemizer charitable deduction ($24,850 total), and it subtracted a $2,000 CTC instead of the 2026 amount of $2,200. Those two errors add about $300 to the correct $9,579.77."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its AGI of about $89,951 equals the correct $117,661 minus another $27,711. That means it deducted the entire $30,711 net capital loss instead of the $3,000 annual limit under IRC §1211(b), understating taxable income by $27,711."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used the correct $117,662 AGI, $24,150 standard deduction, 2026 brackets, $2,200 CTC, and $600 CDCC. It missed the 2026 non-itemizer charitable deduction that takes the $700 cash donation off taxable income, so its $9,734 is $154 (22% of $700) too high."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It took the standard deduction but left out the $700 non-itemizer charitable deduction (worth $154). The rest of its roughly $656 overstatement comes from a 12%/22% breakpoint below the 2026 HoH threshold of $67,450."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized by counting the $21,208 employer-sponsored insurance premiums as medical expenses above the 7.5% floor and adding NY income tax to SALT. The ESI premiums are not a deductible out-of-pocket medical expense here. Without them itemized deductions fall below $24,150, so the correct path is the standard deduction plus the $700 non-itemizer charitable deduction, which gives $92,811 of taxable income."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used the $24,150 standard deduction, the $3,000 loss cap, 2026 brackets, and $2,800 of credits correctly, but missed the 2026 non-itemizer charitable deduction for the $700 cash donation. Taxable income stayed $700 too high, adding $154 of tax at 22% and giving $9,734."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $93,504 taxable income left out the $700 non-itemizer charitable deduction and took the $8 IRA deduction, which the head's 401(k) active-participant status removes at this AGI. Taxable income came out $693 too high, giving $9,732.20 instead of $9,579.77."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It itemized by treating the $21,208 ESI premiums as deductible medical expenses and adding NY income tax to SALT, which overstated deductions. Without the premiums, itemized deductions fall below $24,150, so the correct deduction is the standard deduction plus the $700 non-itemizer charitable deduction, for taxable income of $92,811."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"Its $9,717.24 is $137 above the correct figure, which means taxable income about $625 too high. That fits leaving out most of the $700 non-itemizer charitable deduction that applies alongside the $24,150 HoH standard deduction."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no working. Its $14,000 is above even the correct $12,379.77 pre-credit tax, so it never subtracted the $600 CDCC and $2,200 CTC from a correctly computed liability on $92,811 of taxable income."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: two personal exemptions, a reduced standard deduction, and a $1,000 CTC fully phased out at the old $75,000 HoH threshold. It also subtracted the $21,208 ESI premiums from wages, giving AGI of $96,454. The correct AGI is $117,661, there are no exemptions in 2026, and the full $2,200 CTC applies."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-sunset law: two $5,300 exemptions, 10/15/25% HoH brackets, and a $1,000 CTC phased out above $75,000. OBBBA made TCJA permanent, so 2026 uses no exemptions, the $24,150 standard deduction plus $700 charitable, 10/12/22% brackets, and a $2,200 CTC that is not phased out below $200,000."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It reached $28,094 of itemized deductions by counting the ESI premiums as medical expenses above the 7.5% floor and adding NY income tax to SALT under the expanded cap. Without the premiums, itemizing loses to the $24,150 standard deduction plus the $700 non-itemizer charitable deduction, so taxable income is $92,811, not $89,500."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used post-TCJA-expiration parameters with a $12,591 HoH standard deduction and left out the CTC entirely, subtracting only the $600 CDCC. Under OBBBA the 2026 HoH standard deduction is $24,150 plus the $700 charitable deduction, and the $2,200 CTC is fully nonrefundable against this liability."
-us,scenario_082,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It left out the $700 non-itemizer charitable deduction and took the $8 IRA deduction, which the active-participant phase-out removes, so taxable income came to $93,504 instead of $92,811. Its $10,638 ordinary tax on $80,080 is also about $120 above what the 2026 HoH brackets give ($10,518.60)."
-us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or reasoning was returned for federal_income_tax_before_refundable_credits, so there is no answer to score."
-us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated the capital loss as wiping out the tax, but a net capital loss offsets ordinary income only up to $3,000 (IRC §1211(b)). With AGI of $117,661 and taxable income of $92,811, tax before credits is $12,379.77, far more than the $2,800 of nonrefundable credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It left out the $700 non-itemizer charitable deduction (worth $154). The rest of its roughly $500 overstatement comes from a 12%/22% breakpoint below the 2026 HoH threshold of $67,450 (10% bracket ends at $17,700)."
-us,scenario_082,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It treated the $21,208 employer-sponsored insurance premiums as pretax and removed them from wages. Wages are reduced only by the $181 401(k) deferral, so taxable income is $92,811, not about $71,600 ($92,811 minus $21,208). That error drove the understated $7,436 pre-credit tax."
-us,scenario_082,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its AGI ($117,662), $24,150 standard deduction, brackets, and $2,800 of credits were all right, but it missed the 2026 non-itemizer charitable deduction for the $700 cash donation. Taxable income stayed at $93,512 instead of $92,811, adding $154 of 22% tax."
-us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It netted the full -$30,711 capital loss into AGI instead of capping it at $3,000, and it treated only $400 of the CTC as nonrefundable when the full $2,200 is absorbed by the liability. It then submitted $12,866.90, which does not follow from its own final working of $6,509."
-us,scenario_082,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It used single filing status with a $15,750 standard deduction instead of head of household with $24,150 plus $700 charitable. It claimed a $6,237 QBI deduction with no business income, started from an inflated AGI of $139,775 that skipped proper capital-loss netting, and applied only $1,875 of CTC instead of $2,200 plus the $600 CDCC."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own working left out the $700 non-itemizer charitable deduction, took the $8 IRA deduction, used a $2,000 CTC instead of $2,200, and used bracket edges of $17,300/$65,900 instead of $17,700/$67,450, arriving at about $10,353. It then submitted $14,748, a number its own reasoning never produced."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It deducted the $8 traditional IRA contribution, which is fully phased out for an active 401(k) participant with $117k of AGI. It also miscomputed the HoH bracket tax: it got $10,444.60 on $79,380 instead of $10,364.60, an $80 error equal to putting the top of the 12% bracket about $800 below $67,450."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It deducted the $8 traditional IRA contribution even though the head actively participates in a 401(k) and the AGI is above the 2026 HoH active-participant phaseout, so no IRA deduction is allowed. That understated taxable income ($92,804 instead of $92,811.12) and the tax by about $1.57."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"It filed as single with a $14,600 standard deduction instead of head of household with $24,150 plus the $700 non-itemizer charitable deduction. It applied 20% to the full $9,600 of childcare instead of the $3,000 one-child cap and used only $1,600 of CTC. It then submitted $23,437, which none of its steps support."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave no derivation, and its reasoning mentions only the CDCC, leaving out the $2,200 nonrefundable CTC. The correct calculation is $12,379.77 of tax before credits minus $2,800 of credits, so the $14,500 guess overstates even the pre-credit tax."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a $23,500 HoH standard deduction and old bracket edges ($17,000/$64,850) instead of 2026's $24,150 and $17,700/$67,450. It also left out the $700 non-itemizer charitable deduction and the $68 refund, so taxable income came out at $94,094 instead of $92,811.12."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used a $23,000 HoH standard deduction, pre-2026 brackets ($17,000/$64,850), and a $2,000 CTC, and left out the $700 non-itemizer charitable deduction. That gave $10,460, which it then arbitrarily raised to $11,150."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used a $24,500 standard deduction instead of $24,150 and left out the $700 non-itemizer charitable deduction. It taxed ordinary income with old HoH bracket edges ($17,000/$64,850) instead of $17,700/$67,450, which overstated the ordinary tax."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It explicitly assumed no active-participant phaseout and deducted the $8 traditional IRA contribution. Because the head is in a 401(k) and AGI is about $117.7k, the IRA deduction is fully phased out, so taxable income is $92,811.12, not $92,804."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It built AGI without the $13,424 of qualified dividends and counted the $2,289 short-term gain separately even though it is already netted into the capital loss, giving $106,519 instead of $117,661. It still subtracted the qualified dividends from taxable income, which understated ordinary income by about $8,800. It also used a $22,500 standard deduction and a $2,000 CTC."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its working used a $22,500 standard deduction, 2025 brackets, and a $2,000 CTC, and left out the $700 charitable deduction, reaching $10,739. It then submitted $13,920, which does not follow from any step it showed."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got AGI ($117,662) and the $24,150 standard deduction right but left out the 2026 non-itemizer charitable deduction for the $700 cash donation. That left taxable income at $93,512 instead of $92,811 and overstated the tax by 22% × $700 = $154."
+us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted the full $33,000 long-term loss and also a separate $3,000 capital-loss deduction instead of capping the net loss at $3,000. It used a made-up $46,000 HoH standard deduction and computed the CDCC as 35% of $5,000 and the CTC as $300, instead of $600 and $2,200."
+us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, applying personal exemptions, 10/15/25% brackets, and a $1,000 CTC. In 2026 the $24,150 HoH standard deduction, the 10/12/22% brackets, and the $2,200 CTC all still apply, with no personal exemptions."
+us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied pre-TCJA rules for 2026: personal exemptions, pre-TCJA rate brackets, and a $1,000 CTC phased out to zero. Under the law in effect, the $2,200 CTC is fully available below the $200,000 phaseout threshold, and the 2026 standard deduction plus $700 charitable deduction applies."
+us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $23,100 HoH standard deduction instead of $24,150, left out the $700 non-itemizer charitable deduction, and used a $2,300 CTC instead of $2,200. Its ordinary tax of $11,059.60 on $81,130 reflects outdated bracket edges, not 2026's $17,700/$67,450."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, taking $10,000 of personal exemptions, itemizing $18,564 of SALT, and limiting the CTC to $1,000. Under 2026 law there are no personal exemptions, and the $24,150 standard deduction plus $700 charitable deduction and the $2,200 CTC apply."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It put AGI at $100,571, leaving out most of the $20,480 of dividends instead of reaching $117,661. It subtracted only a $2,000 CTC, leaving out the $600 CDCC and $200 of the $2,200 CTC."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $21,208 of ESI premiums from wages, giving AGI of $96,446; these premiums do not reduce IRS employment income, so AGI is $117,661. It also took pre-TCJA personal exemptions and dropped the $2,200 CTC entirely, subtracting only the $600 CDCC."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It cut AGI to $96,454 by treating the $21,208 of ESI premiums as a pre-tax wage reduction, then applied post-sunset personal exemptions and a $1,000 CTC. The correct calculation keeps AGI at $117,661 and uses the $24,150 + $700 deductions and the $2,200 CTC."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no computation, and its $12,790.25 sits about at the correct pre-credit tax of $12,379.77. That matches failing to subtract the $2,800 of nonrefundable credits ($2,200 CTC plus $600 CDCC)."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It applied personal exemptions, which do not exist in 2026, and left the CTC out of its listed credits. The extra exemptions shrank taxable income well below the correct $92,811.12, giving $7,380 instead of $9,579.77."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and its $3,384 is about $6,200 below the correct result. That is the shortfall produced by deducting the entire $30,711 net capital loss instead of the $3,000 annual cap, which pushes the qualified dividends into the 0% bracket."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It subtracted a $2,000 CTC instead of the 2026 amount of $2,200. Its implied pre-credit tax of about $12,855 is $475 above $12,379.77, which matches leaving out the $700 non-itemizer charitable deduction and using pre-2026 bracket and standard-deduction amounts."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, using an ~$11,900 standard deduction, personal exemptions, $10,700 of itemized deductions, and a $1,000 CTC. Under 2026 law the HoH standard deduction is $24,150 plus the $700 charitable deduction and the CTC is $2,200."
+us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It estimated a $23,063 standard deduction and bracket edges of $16,913/$64,575 instead of 2026's $24,150 and $17,700/$67,450. It used a $2,000 CTC instead of $2,200, and it left out the $700 non-itemizer charitable deduction and the $68 refund."
+us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It counted the $21,208 of ESI premiums as deductible medical expenses and itemized $23,266. Actual out-of-pocket medical costs ($1,965) are below the 7.5% AGI floor, and $23,266 is less than the $24,150 standard deduction plus $700 charitable deduction in any case. It then taxed $94,396 instead of $92,811.12 using outdated bracket edges."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave no figures. Its $12,719 roughly equals the correct pre-credit tax of $12,379.77, which matches failing to net the $2,200 CTC and $600 CDCC against the tax."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It itemized real estate tax, medical, and charity instead of taking the larger $24,150 standard deduction plus $700 charitable deduction. Its $17,303 is above even the $12,379.77 pre-credit tax, so it also failed to apply the $3,000 capital-loss deduction, the 15% qualified-dividend rate, and the $2,800 of credits correctly."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It itemized $24,415, which is less than the $24,850 available from the $24,150 standard deduction plus the $700 non-itemizer charitable deduction. It also used a $2,000 CTC instead of $2,200, overstating the tax by about $317."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $89,951 AGI is $27,710 below the correct $117,661, exactly the part of the $30,711 net capital loss above the $3,000 annual limit. In other words, it deducted the whole capital loss instead of carrying $27,711 forward."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used the right AGI ($117,662), standard deduction, and credits but left out the 2026 non-itemizer charitable deduction for the $700 donation. That overstated taxable income by $700 and the tax by $154."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"After the correct $2,200 CTC and $600 CDCC, its answer implies pre-credit tax of about $13,036, $656 above $12,379.77. That matches leaving out the $700 non-itemizer charitable deduction and taxing ordinary income with pre-2026 bracket thresholds or a smaller standard deduction."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized medical expenses on top of SALT and charity, which only works by counting the $21,208 of ESI premiums as out-of-pocket medical costs. Actual medical spending ($1,965) is below the 7.5% AGI floor, so the $24,150 standard deduction plus the $700 non-itemizer charitable deduction is the correct choice."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It took only the HoH standard deduction and left out the 2026 non-itemizer charitable deduction for the $700 cash donation. Taxable income was overstated by $700 and the tax by $154 ($9,734 instead of $9,579.77)."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $93,504 taxable income comes from deducting the $8 IRA contribution, which is phased out for an active 401(k) participant, and leaving out the $700 non-itemizer charitable deduction. Correct taxable income is $92,811.12."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It itemized a medical deduction, which requires treating the $21,208 of ESI premiums as out-of-pocket medical spending. Actual medical costs ($1,965) are below the 7.5% AGI floor, so the $24,150 standard deduction plus the $700 non-itemizer charitable deduction applies and taxable income is $92,811.12."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"Its $137 overstatement equals 22% of about $625 of extra taxable income. That is the missing $700 non-itemizer charitable deduction, partly offset by excluding the $68 refund and deducting the phased-out $8 IRA contribution."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave only a round $14,000 guess with no derivation. That is above even the correct $12,379.77 pre-credit tax, which matches not applying the $2,200 CTC and $600 CDCC and overstating the base tax."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA had expired, applying personal exemptions and treating the CTC as fully phased out, when the $2,200 CTC is fully available at this income. It also subtracted the $21,208 of ESI premiums from wages, understating AGI ($96,454 instead of $117,661)."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-sunset law: two $5,300 exemptions, 10/15/25% HoH brackets, and a $1,000 CTC phased out above $75,000. Under 2026 law, the 10/12/22% brackets, the $24,150 + $700 deductions, and the full $2,200 CTC apply."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Its $28,094 of itemized deductions includes a medical deduction that only exists if the $21,208 of ESI premiums counts as out-of-pocket medical spending. Actual medical costs ($1,965) are below the 7.5% floor, so the $24,150 standard deduction plus the $700 charitable deduction applies. It also left out the $68 refund."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a post-TCJA-sunset standard deduction of $12,591 and pre-TCJA brackets, giving taxable income of $105,063 instead of $92,811.12. It dropped the $2,200 CTC entirely, subtracting only the $600 CDCC."
+us,scenario_082,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It deducted the phased-out $8 IRA contribution and left out the $700 non-itemizer charitable deduction, giving taxable income of $93,504. It also overstated the ordinary tax ($10,638 instead of about $10,519 on its own base) by misapplying the 2026 HoH bracket edges of $17,700/$67,450."
+us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no value and no explanation for federal_income_tax_before_refundable_credits.
+us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It let the capital loss wipe out the tax liability, ignoring the $3,000 annual limit on deducting net capital losses against ordinary income. With only $3,000 deductible, taxable income is $92,811.12 and the tax after credits is $9,579.77, not zero."
+us,scenario_082,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"After the correct $2,200 CTC and $600 CDCC, its answer implies pre-credit tax of $13,030, $650 above $12,379.77. That matches leaving out the $700 non-itemizer charitable deduction and using pre-2026 bracket thresholds or standard-deduction amounts."
+us,scenario_082,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It treated the $21,208 of ESI premiums as a pre-tax payroll exclusion, cutting taxable income to about $71,600. These premiums do not reduce IRS employment income: wages are reduced only by the $181 traditional 401(k) deferral, so taxable income is $92,811.12."
+us,scenario_082,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Its AGI, standard deduction, brackets, and credits are all right, but it left out the 2026 non-itemizer charitable deduction for the $700 cash gift. Taxable income is overstated by $700 and the tax by $154."
+us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It subtracted the full $30,711 net capital loss instead of the $3,000 cap, used a $21,900 standard deduction, and treated only $400 of the CTC as nonrefundable. Its own working ended at $6,509, yet it submitted an unrelated $12,866.90."
+us,scenario_082,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"It filed as single with a $15,750 standard deduction instead of head of household with $24,150 plus $700. It added a QBI deduction even though there is no business income, inflated AGI to $139,775, and used a $1,875 CTC while leaving out the $600 CDCC."
us,scenario_082,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It applied only the ACTC limitation formula — lesser of $1,700 and 15% of earned income over $2,500 — and never performed the ordering step in which the nonrefundable CTC is first applied against tax liability and only the unused remainder becomes refundable. Its own reasoning conceded the head 'has sufficient nonrefundable CTC capacity' and then discarded that conclusion; with roughly $117,650 of AGI the pre-credit federal tax absorbs the entire credit, leaving $0. It also used the pre-OBBBA $2,000 per-child credit rather than the 2026 $2,200 amount."
us,scenario_082,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It reported the $1,700 statutory refundable ceiling as the answer after checking only the $2,500 earned-income floor, skipping the requirement that the refundable ACTC equals the CTC remaining after the nonrefundable portion offsets tax liability. Even at its own stated AGI of about $86,949, head-of-household tax before credits far exceeds the $2,200 CTC, so the credit is fully used nonrefundably and refundable credits are $0."
us,scenario_082,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"It quoted $1,000, the pre-TCJA additional child tax credit cap, which has no basis in 2026 law where the credit is $2,200 per child with a $1,700 refundable ceiling. The controlling error is treating a refundable amount as automatic: this filer's federal tax before credits exceeds the full $2,200 credit, so the nonrefundable CTC absorbs all of it and $0 is refundable."
@@ -7417,156 +7495,159 @@ us,scenario_082,local_income_tax,gemini-3.1-flash-lite-preview,llm_error,state_l
us,scenario_082,local_income_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"It applied NYC head-of-household brackets to an NYC taxable income of $79,254, treating the household as NYC residents when only state = NY was given. Because NYC residence is an unlisted status that defaults to false, no NYC tax arises and local income tax is $0."
us,scenario_082,local_income_tax,glm-5.2,llm_error,state_local_rule,False,"It said outright that it 'assumed to reside in NYC (the only NY locality with a local income tax),' which breaks the instruction to treat unlisted facts as false. Without NYC residence, its whole NYC bracket, school tax credit and child care credit computation does not apply, and local income tax is $0."
us,scenario_082,local_income_tax,inkling,llm_error,state_local_rule,False,"It stated that 'NYC resident income tax applies' to NY taxable income of about $104,454 and applied NYC brackets, but the household is never placed in NYC. Under the default-false rule for unlisted status facts, the household owes no NYC tax and local income tax is $0."
-us,scenario_082,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"Noticed NY PFL/DBL but left them out, stopping at federal FICA of $7,664.92. Reading 1 counts the NY employee maximums (PFL $411.91 plus DBL $31.20) for $8,108.03. Reading 2 treats them as employer-optional, not mandatory employee taxes, for $7,664.92. The model chose reading 2."
-us,scenario_082,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"Counted only federal SS ($6,212.09) and Medicare ($1,452.83), which matches the reading where NY's employer-optional PFL/DBL contributions are not mandatory employee taxes. The reference reading adds PFL at 0.432% capped at $411.91 and DBL of $31.20, giving $8,108.03."
-us,scenario_082,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"Used a 0.62% NY PFL rate with no cap ($621.21), then added unnamed 'other' NY withholdings to reach $8,304.92. The 2026 PFL rate is 0.432% with a $411.91 annual maximum, and the only other employee contribution is DBL at $31.20."
-us,scenario_082,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Rounded federal FICA to $7,665 and left out NY contributions, which matches the reading where employer-optional NY PFL/DBL are not mandatory employee taxes. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"Included DBL correctly at $31.20, but cut the NY PFL to a made-up $49.31 after first estimating $354–$389 at the 2025 rate of 0.388%. The 2026 PFL is 0.432% of wages capped at $411.91, so NY adds $443.11, not $80.51."
-us,scenario_082,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"Said NY has no mandatory employee payroll tax and stopped at $7,664.92, which matches the reading where employer-optional PFL/DBL are excluded. The reference reading counts NY PFL (0.432%, capped at $411.91) and DBL ($31.20) as employee contributions for $8,108.03."
-us,scenario_082,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Left out NY employee contributions and rounded federal FICA to $7,665, following the reading where NY PFL/DBL are not mandatory employee taxes. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"Treated NY as having no modeled employee payroll tax and gave $7,664.92, following the employer-optional reading of NY PFL/DBL. The reference reading counts PFL $411.91 and DBL $31.20 as mandatory employee contributions, for $8,108.03."
-us,scenario_082,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"Got DBL right ($31.20) but figured the PFL with the 2025 rate of 0.388% on an old ~$89,343 wage cap ($346.65). The 2026 PFL is 0.432% of wages capped at $411.91, so the model is $65.26 short."
-us,scenario_082,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"Added an arbitrary $129.39 for NY SDI/PFL. The 2026 statutory amounts are PFL $411.91 (0.432%, capped) and DBL $31.20 ($0.60/week), a NY total of $443.11."
-us,scenario_082,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"Said no NY payroll tax is modeled and gave federal FICA of $7,664.92, following the employer-optional reading of NY PFL/DBL. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"Took the $181 traditional 401(k) deferral out of FICA wages. Under IRC §3121(v)(1)(A), elective deferrals stay subject to Social Security and Medicare, so FICA applies to the full $100,195. The model also left out NY PFL ($411.91) and DBL ($31.20)."
-us,scenario_082,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Included the NY DBL contribution ($31.20) but left out the NY Paid Family Leave contribution. PFL is withheld on the same basis at 0.432% of wages capped at $411.91, so the model is short by exactly $411.91."
-us,scenario_082,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"Used a stale NY PFL maximum of $369.68 instead of the 2026 cap of $411.91, and left out the $31.20 DBL contribution entirely."
-us,scenario_082,payroll_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"Applied the correct 0.432% PFL rate but to all $100,195 of wages ($432.84) without the annual cap. The 2026 PFL employee contribution is capped at $411.91, so the model is $20.93 over."
-us,scenario_082,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"Used about $333 for NY PFL, which is the old 2024 maximum. The 2026 PFL is 0.432% capped at $411.91. DBL ($31) was roughly right."
-us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"Applied only 7.65% FICA to $100,195 ($7,665), which matches the reading where NY's employer-optional PFL/DBL are not mandatory employee taxes. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Took the $21,208 ESI premium out of FICA wages ($78,987), assuming a Section 125 pre-tax deduction that the facts do not give, and also figured the PFL on the reduced wages at an old rate ($294.62). FICA applies to the full $100,195 ($7,664.92), and PFL is capped at $411.91."
-us,scenario_082,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"Lowered FICA wages to $78,987 by subtracting ESI premiums without any stated pre-tax cafeteria-plan arrangement, which cut SS/Medicare to $6,042.50. It also left out the $31.20 DBL and used about $394 instead of $411.91 for PFL."
-us,scenario_082,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"Counted only federal SS and Medicare on gross wages ($7,664.92), following the reading where NY PFL/DBL are employer-optional, not mandatory employee taxes. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"Used ESI-reduced FICA wages of $78,987, a pre-tax premium deduction the facts do not give. FICA should apply to all $100,195. Its NY PFL/DBL amount (about $270) is also below the 2026 total of $443.11."
-us,scenario_082,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Subtracted the $21,208 ESI premium from FICA wages ($78,987) without any stated pre-tax arrangement, which understates SS/Medicare by $1,622. Its NY add-on (about $333) is also below PFL $411.91 plus DBL $31.20."
-us,scenario_082,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"Gave only federal SS ($6,212.09) and Medicare ($1,452.83), following the reading where NY's employer-optional PFL/DBL are excluded. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"Named the right components (FICA $7,664.92, DBL $31.20, PFL about $412.80) but added them wrong: they sum to $8,108.92, not the $8,096.92 reported. The PFL should also be the exact 2026 cap of $411.91."
-us,scenario_082,payroll_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"Figured NY PFL at the 2025 rate of 0.388% on a $96,200 cap ($373.26). The 2026 rate is 0.432% with a $411.91 annual maximum. DBL at $31.20 was correct."
-us,scenario_082,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"Its own components, SS $6,212 plus Medicare $1,453, total $7,665, but it reported $7,765, a $100 addition error. It also left out NY PFL ($411.91) and DBL ($31.20), which the reference counts as employee payroll taxes."
-us,scenario_082,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Left out NY employee contributions on the reading that NY has no mandatory employee payroll tax, since employers can fund PFL/DBL. It also rounded federal FICA down to $7,660 from $7,664.92. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"Counted only SS and Medicare, following the employer-optional reading of NY PFL/DBL, and rounded down to $7,659 from $7,664.92. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,"Its NY total of $385.73 is the 2025 PFL maximum ($354.53) plus DBL ($31.20). The 2026 PFL maximum is $411.91, so the NY total should be $443.11."
-us,scenario_082,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"Included the NY disability contribution ($31.20) but left out the NY Paid Family Leave contribution, which is withheld alongside it at 0.432% of wages capped at $411.91. The model is short by exactly that amount."
-us,scenario_082,payroll_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"Applied 7.65% FICA only ($7,664.92), following the reading where NY's employer-optional PFL/DBL are not mandatory employee taxes. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Left out NY PFL/DBL on the employer-optional reading and also misstated 7.65% of $100,195 as $7,654 instead of $7,664.92. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"Applied the 0.432% PFL rate to all wages ($432.84) and missed the 2026 annual cap of $411.91, overstating PFL by $20.93. DBL at $31.20 was correct."
-us,scenario_082,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"Applied only 7.65% FICA ($7,665), following the reading where NY PFL/DBL are employer-optional and not mandatory employee taxes. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"Took the $21,208 ESI premium out of FICA wages ($78,987) with no stated pre-tax cafeteria-plan election, which understates SS/Medicare. It also said NY has no employee payroll tax and left out PFL $411.91 and DBL $31.20."
-us,scenario_082,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"Used about $367 for NY PFL instead of the 2026 cap of $411.91, and explicitly left out the $31.20 NY DBL employee contribution."
-us,scenario_082,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"Counted only federal SS and Medicare ($7,665), following the employer-optional reading of NY PFL/DBL. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"Took both the $21,208 ESI premium and the $181 traditional 401(k) deferral out of FICA wages ($78,806). 401(k) deferrals are FICA wages under IRC §3121(v), and no pre-tax premium arrangement is given, so FICA applies to $100,195. The model also left out NY PFL/DBL."
-us,scenario_082,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"Estimated NY PFL at about $365 (an older rate and cap) instead of the 2026 cap of $411.91 (0.432% of wages). DBL at about $31 was right, so the NY total is $396 instead of $443.11."
-us,scenario_082,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value and no explanation for payroll_tax, so there was no answer to score against $8,108.03."
-us,scenario_082,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"Counted only SS ($6,212.09) and Medicare ($1,452.83), rounded to $7,665, which matches the reading where NY PFL/DBL are employer-optional. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,muse-spark-1.3,prompt_ambiguity,state_local_rule,False,"Put NY PFL plus DBL at about $380, which matches stale pre-2026 PFL caps. The 2026 amounts are PFL $411.91 (0.432%, capped) plus DBL $31.20, or $443.11."
-us,scenario_082,payroll_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"Applied FICA to $78,987 after taking out ESI premiums as if paid pre-tax, which the facts do not state, so SS/Medicare came to $6,042.50 instead of $7,664.92 on full wages. Its NY add-on of $372.40 is also below $443.11."
-us,scenario_082,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"Counted only federal SS and Medicare ($7,665), following the reading where NY's employer-optional PFL/DBL are not mandatory employee taxes. The reference reading adds PFL $411.91 and DBL $31.20 for $8,108.03."
-us,scenario_082,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"Dropped its own reasonable intermediate totals ($8,038–$8,169) and subtracted a made-up '-$200.61 NY deduction' from FICA. NY employee contributions are positive additions: PFL $411.91 plus DBL $31.20."
-us,scenario_082,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"Its stated components (SS $6,212.09 plus Medicare $1,452.83, no state tax) total $7,664.92, but it reported $7,752.42, an $87.50 addition error. It also left out NY PFL ($411.91) and DBL ($31.20)."
+us,scenario_082,payroll_tax,claude-fable-5,prompt_ambiguity,state_local_rule,False,"Computed FICA correctly at $7,664.92 but excluded New York's PFL and DBL as negligible. That is the reading where New York employee contributions are not mandatory because employers can absorb them. The reference's reading includes the statutory maximum: $411.91 PFL (0.432%, capped) plus $31.20 DBL."
+us,scenario_082,payroll_tax,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($6,212.09 + $1,452.83) and never added New York employee PFL or DBL. That is the reading where employer-absorbable New York contributions are not 'mandatory.' The reference instead adds the maximum $411.91 PFL and $31.20 DBL, for $443.11."
+us,scenario_082,payroll_tax,claude-fable-5.2,prompt_ambiguity,state_local_rule,False,"Explicitly left out any New York employee payroll tax and stopped at $7,664.92 of FICA. The reference reads 'mandatory employee state payroll taxes' as including New York's maximum PFL ($411.91) and DBL ($31.20) withholding, while the model read it as excluding contributions an employer can absorb."
+us,scenario_082,payroll_tax,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"Applied a 0.62% PFL rate to all $100,195 of wages ($621.21) with no annual cap, then added about $18.79 of unspecified 'other withholdings.' The 2026 PFL employee contribution is 0.432% capped at $411.91, and DBL is $31.20, so New York adds $443.11, not $640."
+us,scenario_082,payroll_tax,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"Reported rounded FICA only ($7,665) and omitted New York's PFL and DBL employee contributions. This follows the reading that employer-absorbable New York contributions are not mandatory. The reference instead includes the $411.91 PFL cap and $31.20 DBL."
+us,scenario_082,payroll_tax,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"Correctly set DBL at $31.20 and first estimated PFL near $389. It then swapped in an unsupported $49.31 PFL figure, so it added only $80.51 for New York. The 2026 PFL is 0.432% of wages capped at $411.91, so New York payroll tax is $443.11."
+us,scenario_082,payroll_tax,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"Said New York has no mandatory employee payroll tax and reported FICA of $7,664.92. That matches the reading where New York PFL/DBL contributions are excluded because employers can absorb them. The reference adds the statutory maximum: $411.91 PFL plus $31.20 DBL."
+us,scenario_082,payroll_tax,claude-opus-5,prompt_ambiguity,state_local_rule,False,"Reported FICA only, rounded to $7,665, and said New York has no counted employee payroll tax. The reference treats New York's PFL ($411.91 at the 0.432% cap) and DBL ($31.20) as mandatory employee state payroll taxes. The model took the other reading, where employer-absorbable contributions are excluded."
+us,scenario_082,payroll_tax,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"Computed $7,664.92 of FICA and said no New York employee payroll tax applies. The reference instead includes New York's maximum employee PFL ($411.91) and DBL ($31.20). Both readings of 'mandatory employee state payroll taxes' are available because New York lets employers absorb those contributions."
+us,scenario_082,payroll_tax,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"Included DBL correctly at $31.20 but priced PFL with the outdated 0.388% rate on a ~$89,343 wage cap, giving $346.65. The 2026 PFL employee contribution is 0.432% capped at $411.91, which leaves the model's total $65.26 short."
+us,scenario_082,payroll_tax,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"Added an arbitrary ~$129.39 for New York SDI/PFL on top of $7,664.92 of FICA. The correct New York employee amounts are $411.91 PFL (0.432% capped) plus $31.20 DBL, for $443.11."
+us,scenario_082,payroll_tax,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"Reported $7,664.92 of FICA and said no New York payroll tax is modeled. That is the reading where New York PFL/DBL is optional for employers to collect. The reference instead includes the maximum $411.91 PFL and $31.20 DBL employee contributions."
+us,scenario_082,payroll_tax,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"Subtracted the $181 traditional 401(k) deferral from FICA wages, giving $100,014. Elective 401(k) deferrals stay subject to FICA under IRC §3121(v)(1)(A), so FICA applies to the full $100,195. The model also omitted New York's $411.91 PFL and $31.20 DBL."
+us,scenario_082,payroll_tax,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"Added New York DBL ($31.20) but left out the New York Paid Family Leave employee contribution. At 0.432% of wages, that contribution hits its $411.91 cap, so the total falls $411.91 short."
+us,scenario_082,payroll_tax,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,Used an outdated New York PFL maximum of $369.68 instead of the 2026 cap of $411.91 (0.432% rate). It also omitted the $31.20 New York DBL contribution.
+us,scenario_082,payroll_tax,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,Applied the 0.432% PFL rate to all wages ($432.84) without the $411.91 annual maximum. That overstates PFL by $20.93; DBL ($31.20) and FICA were correct.
+us,scenario_082,payroll_tax,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"Used a stale PFL amount of about $333 (an earlier year's cap) instead of the 2026 maximum of $411.91 at 0.432%, and rounded DBL to $31. As a result it understated New York contributions by about $79."
+us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"Applied 7.65% FICA to $100,195 ($7,665) and added nothing for New York. That is the reading where employer-absorbable New York PFL/DBL is not mandatory. The reference includes the maximum $411.91 PFL and $31.20 DBL."
+us,scenario_082,payroll_tax,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"Treated the $21,208 ESI premium input as a pre-tax reduction of FICA wages, cutting the base to $78,987, when the engine applies FICA to the full $100,195. It also understated PFL at $294.62 instead of the $411.91 cap."
+us,scenario_082,payroll_tax,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"Took the $21,208 ESI premiums out of FICA wages, leaving $78,987, instead of taxing the full $100,195 of gross wages. It also put PFL at ~$394 rather than the $411.91 cap and omitted the $31.20 DBL."
+us,scenario_082,payroll_tax,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,664.92) with no New York contributions. That follows the reading where New York PFL/DBL is not mandatory because the employer can absorb it. The reference adds the maximum $411.91 PFL and $31.20 DBL."
+us,scenario_082,payroll_tax,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"Computed FICA on $78,987 after subtracting ESI premiums as pre-tax deductions, but the engine applies FICA to the full $100,195. That alone understates FICA by about $1,622, and the model's ~$270 New York add-on is below the $443.11 maximum."
+us,scenario_082,payroll_tax,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"Reduced FICA wages to $78,987 by subtracting the $21,208 ESI premium, but FICA applies to the full $100,195 of gross wages. Its ~$333 New York add-on also falls short of the $411.91 PFL + $31.20 DBL."
+us,scenario_082,payroll_tax,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"Summed only Social Security and Medicare ($7,664.92) and left out New York PFL and DBL. That reading treats employer-absorbable New York contributions as not mandatory. The reference instead includes the statutory maximum $411.91 PFL and $31.20 DBL."
+us,scenario_082,payroll_tax,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"Included both New York components (PFL ~$412.80, DBL $31.20) but added $7,664.92 + $31.20 + $412.80 as $8,096.92 instead of $8,108.92, a $12 addition error. Its PFL figure also slightly exceeds the $411.91 cap."
+us,scenario_082,payroll_tax,glm-5.2,prompt_ambiguity,state_local_rule,False,"Priced PFL at the outdated 0.388% rate on a ~$96,200 cap, giving $373.26. The 2026 rate is 0.432% with a $411.91 maximum, so PFL is $38.65 short; DBL ($31.20) and FICA were correct."
+us,scenario_082,payroll_tax,glm-5.3,prompt_ambiguity,state_local_rule,False,"Listed FICA components of $6,212 + $1,453, which sum to $7,665, but reported $7,765, a $100 addition error. It also said New York has no employee payroll tax and omitted the $411.91 PFL and $31.20 DBL."
+us,scenario_082,payroll_tax,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"Excluded New York PFL/DBL on the reading that New York has no mandatory employee payroll tax. The reference includes the employer-collectible maximum of $411.91 PFL and $31.20 DBL. The model also misstated FICA as $7,660 instead of $7,664.92."
+us,scenario_082,payroll_tax,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"Reported only federal FICA and dropped New York's PFL ($411.91) and DBL ($31.20). That fits the reading where those contributions are not mandatory because the employer can absorb them. It also rounded FICA down to $7,659 instead of $7,664.92."
+us,scenario_082,payroll_tax,gpt-5.5,prompt_ambiguity,state_local_rule,False,Estimated New York PFL/DBL at $385.73 instead of $443.11 ($411.91 PFL at the 0.432% cap plus $31.20 DBL). That points to a stale PFL rate or cap.
+us,scenario_082,payroll_tax,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"Added only New York disability ($31.20) and omitted the New York Paid Family Leave employee contribution. At 0.432% of $100,195, that contribution is capped at $411.91."
+us,scenario_082,payroll_tax,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"Applied 7.65% FICA for $7,664.92 and added no New York payroll tax. That is the reading where New York PFL/DBL is excluded because employers can absorb it. The reference includes the maximum $411.91 PFL and $31.20 DBL."
+us,scenario_082,payroll_tax,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"Reported FICA only and omitted New York's $411.91 PFL and $31.20 DBL, following the reading that employer-absorbable New York contributions are not mandatory. It also misstated FICA as $7,654 instead of $7,664.92."
+us,scenario_082,payroll_tax,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"Used uncapped PFL of 0.432% × $100,195 = $432.84 instead of the 2026 annual maximum of $411.91, overstating the total by $20.93."
+us,scenario_082,payroll_tax,grok-4.3,prompt_ambiguity,state_local_rule,False,"Applied 7.65% FICA only ($7,665) with nothing for New York. That fits the reading where New York PFL/DBL is not mandatory. The reference instead includes the maximum $411.91 PFL and $31.20 DBL."
+us,scenario_082,payroll_tax,grok-4.5,prompt_ambiguity,state_local_rule,False,"Subtracted the $21,208 ESI premium from FICA wages, computing FICA on $78,987, when the engine taxes the full $100,195. It also omitted New York's $411.91 PFL and $31.20 DBL."
+us,scenario_082,payroll_tax,grok-4.6,prompt_ambiguity,state_local_rule,False,"Put PFL at about $367 instead of the 2026 cap of $411.91 (0.432% rate). It also said no other New York employee payroll tax applies, which drops the $31.20 DBL contribution."
+us,scenario_082,payroll_tax,grok-4.7,prompt_ambiguity,state_local_rule,False,"Reported FICA only ($7,665) and never added New York PFL or DBL. That is the reading where employer-absorbable New York contributions are excluded. The reference instead includes the maximum $411.91 PFL and $31.20 DBL."
+us,scenario_082,payroll_tax,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"Reduced FICA wages to $78,806 by subtracting both the ESI premiums and the $181 traditional 401(k) deferral. FICA applies to the full $100,195, and 401(k) deferrals stay FICA wages. The model also omitted New York's $443.11 PFL/DBL."
+us,scenario_082,payroll_tax,inkling,prompt_ambiguity,state_local_rule,False,"Estimated PFL at about $365 instead of the 2026 maximum of $411.91 at 0.432%, and rounded FICA and DBL. As a result its New York add-on is $396 rather than $443.11."
+us,scenario_082,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there was nothing to score against the $8,108.03 reference."
+us,scenario_082,payroll_tax,minimax-m3,prompt_ambiguity,state_local_rule,False,"Computed FICA of $7,664.92, rounded to $7,665, and included no New York contributions. That follows the reading where New York PFL/DBL is not mandatory. The reference adds the $411.91 PFL maximum and the $31.20 DBL."
+us,scenario_082,payroll_tax,muse-spark-1.3,prompt_ambiguity,state_local_rule,False,"Estimated New York PFL/DBL at about $380 instead of $443.11 ($411.91 PFL at the 0.432% cap plus $31.20 DBL), which points to a stale PFL rate."
+us,scenario_082,payroll_tax,muse-wonder,prompt_ambiguity,state_local_rule,False,"Computed FICA on $78,987 after treating ESI premiums as pre-tax, but the engine applies FICA to the full $100,195. Its $372.40 New York estimate also falls short of the $443.11 maximum PFL plus DBL."
+us,scenario_082,payroll_tax,ox-alpha,prompt_ambiguity,state_local_rule,False,"Reported FICA only (about $7,665) with no New York employee contributions. That is the reading where employer-absorbable New York PFL/DBL is excluded. The reference includes the maximum $411.91 PFL and $31.20 DBL."
+us,scenario_082,payroll_tax,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"Its intermediate results were near $8,038, but its final step subtracted about $200 as a negative 'NY mandatory' adjustment, landing below FICA alone. New York PFL ($411.91) and DBL ($31.20) add $443.11 to the $7,664.92 of FICA."
+us,scenario_082,payroll_tax,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"Listed components that sum to $7,664.92 (no state tax) but reported $7,752.42, leaving $87.50 unexplained. It also omitted New York's $411.91 PFL and $31.20 DBL."
us,scenario_082,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_082,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Subtracted about $396 of NY child and dependent care credit as if it were nonrefundable, but the NY CDCC is refundable and is excluded from tax before refundable credits. Its gross tax of $5,891 is also wrong; the correct figure is $5,442.63 main tax plus $156.51 supplemental tax on $105,393.62. It also skipped the $68 SALT-refund subtraction from NY AGI."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"The main bracket tax was right, but the supplemental recapture was measured against a flat 5.4% rate, giving $49.75. The engine phases in the $787 HOH incremental benefit over $50,000 of NY AGI above $107,650, giving $156.51. It also left the $68 SALT refund in NY AGI and deducted the $8 traditional IRA contribution, which is not deductible at this income."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"Used a federal-style taxable income of $62,149, with the $14,600 federal single standard deduction, as the NY tax base. The NY base is NY AGI of $117,593.62 minus the $11,200 HOH standard deduction and the $1,000 dependent exemption. The $62,149 figure doesn't follow from its own AGI inputs, and the $5,649 wasn't computed from any NY rate schedule."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"Gave a rounded guess with no bracket computation. The correct route taxes $105,393.62 on the 2026 HOH schedule ($5,442.63) and adds $156.51 of supplemental tax because NY AGI is above $107,650. The $5,200 fits rough bracket rates with no supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"Filed as single with the $8,000 standard deduction and no $1,000 dependent exemption, when head of household with an $11,200 deduction applies. It also counted the $2,289 short-term gain on top of the $3,000 capped loss, which inflated AGI to $119,943. It then cut its $6,221 single-schedule tax to $5,076 without naming any nonrefundable credit."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"Filed as single with the $8,000 standard deduction and no dependent exemption, and did not take the $181 traditional 401(k) out of wages. It taxed income above $80,650 at a single-schedule 6% rate to get $6,326. It then cut that to $5,276 with no qualifying nonrefundable credit."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Used an $8,000 standard deduction for head of household instead of NY's $11,200 HOH amount and left out the $1,000 dependent exemption. It then applied rough 5.5–6% marginal rates instead of the 2026 HOH schedule, which uses 5.4% from $20,900 to $107,650, plus the $156.51 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Got taxable income right, but added a bracket break at $32,200 and taxed everything above it at 5.9%, giving $5,808.15. The NY HOH 5.4% bracket actually runs from $20,900 to $107,650 and yields $5,442.63. Its $81.40 recapture also understates the $156.51 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"Filed as single and itemized $9,917, when head-of-household status gives an $11,200 standard deduction plus the $1,000 dependent exemption. It then used married-joint bracket thresholds with old rates (5.85%). It also subtracted a $120 NY child care credit, which is refundable and doesn't belong in this measure."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"Filed as single and itemized $9,917 instead of taking the $11,200 HOH standard deduction and $1,000 dependent exemption. It taxed income above $80,650 at 6.25% to get $6,193. It then cut that to $4,800 for unnamed household and minor credits, even though the household credit is fully phased out at this income."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Got taxable income of about $105,394, then applied rough 4–5.5% rates and stopped at $5,400. It left out the $156.51 supplemental recapture tax that applies because NY AGI is above $107,650; the main tax at 2026 rates is $5,442.63."
-us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"Started from federal AGI of about $86,949, roughly $30,700 below the correct $117,661.12, which fits deducting the full net capital loss instead of the $3,000 limit. It claimed $2,000 of exemptions when NY allows only the $1,000 dependent exemption. It also subtracted the Empire State Child Credit, which is refundable and excluded from this measure."
-us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Claimed $20,691 of itemized deductions, but allowable NY itemized deductions are only $9,328.22, so the $11,200 HOH standard deduction applies. It also used wrong HOH bracket widths and the old 5.5% rate, and left out the $156.51 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Subtracted only the $11,200 standard deduction and left out the $1,000 dependent exemption, which overstated taxable income. It then deducted a $120 NY child and dependent care credit, which is refundable and doesn't belong in tax before refundable credits."
-us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Subtracted $2,000 of exemptions, but NY gives no exemption for the filer, only the $1,000 for the dependent, so taxable income is $105,393.62, not $104,386. It also left out the $156.51 supplemental recapture tax triggered by NY AGI above $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Used a $10,500 standard deduction instead of the $11,200 HOH amount and left out the $1,000 dependent exemption, giving $107,086 of taxable income. It subtracted a $30 household credit that is fully phased out at this income, and it left out the $156.51 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Based the tax on wage income only, leaving out the $20,480 of dividends and $100 of interest that are part of NY AGI ($117,593.62). The $4,380 matches taxable income of about $85,700, not $105,393.62, and it has no supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $84,178 of taxable income is about $21,216 below the correct $105,393.62. That gap matches subtracting the $21,208 employer-sponsored insurance premiums, which are not taken out of the listed wages, plus the non-deductible $8 IRA contribution. It also left out the $156.51 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its NY AGI of $96,454 approximately matches federal AGI of $117,661.12 minus the $21,208 employer-paid insurance premiums, which it wrongly took out of income; the engine does not subtract those premiums. It also subtracted $17,200 of deductions and exemptions instead of $12,200 ($11,200 HOH standard deduction plus $1,000 dependent exemption)."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"Gave no computation. The $6,040.50 is $441 too high, which fits using the $8,000 single standard deduction or a single-filer schedule instead of the $11,200 HOH deduction, the $1,000 dependent exemption, and the 2026 HOH 5.4% bracket through $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Gave no computation. At 2026 HOH rates, $4,220 matches taxable income of about $83,000, roughly $22,000 below the correct $105,393.62. That shortfall fits wrongly removing the $21,208 ESI premiums from wages, and the answer also lacks the $156.51 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Gave no computation. At 2026 HOH rates, $3,591 matches taxable income near $71,000, about $34,000 below the correct $105,393.62. That fits deducting the full $30,711 net capital loss instead of the $3,000 limit."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Used the correct $11,200 HOH standard deduction and $1,000 exemption but stopped at the bracket tax. It left out the $156.51 supplemental recapture tax that applies because NY AGI of $117,593.62 is above $107,650. Its $5,533 is also above the $5,442.63 main tax at 2026 rates, which fits using old bracket rates."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"Subtracted the $68 refund twice (its federal AGI of $117,586 already excluded it) and deducted the non-deductible $8 IRA contribution. It then subtracted nonrefundable household and child care credits, even though the household credit is phased out and the NY CDCC is refundable."
-us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"Returned no value for state_income_tax_before_refundable_credits, so there is no answer to score against $5,599.14."
-us,scenario_082,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Itemized $27,966 by counting about $4,700 of NY income tax, which NY requires adding back, and $13,349 of medical expenses, far above the actual $1,965 out-of-pocket. Allowable NY itemized deductions are only $9,328.22, below the $11,200 HOH standard deduction. It also subtracted the refundable $120 NY CDCC."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Gave no computation. The $5,150 is $449 too low, which fits understating the 2026 HOH bracket tax on $105,393.62 ($5,442.63) and leaving out the $156.51 supplemental recapture tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"Gave no computation. The $7,777 is $2,178 too high, which fits applying rates well above the 2026 HOH 5.4% bracket or overstating taxable income by about $35,000. The correct route taxes $105,393.62 on the HOH schedule ($5,442.63) and adds the $156.51 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Got taxable income of $105,394, but applied the pre-2026 HOH rates (4%/4.5%/5.25%/5.5%) to get $5,548, instead of the 2026 rates (3.9%/4.4%/5.15%/5.4%), which give $5,442.63. It also left out the $156.51 supplemental recapture tax triggered by NY AGI above $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Gave no income or deduction figures. The $2,954.80 matches taxable income near $59,000, about $46,000 below the correct $105,393.62. The engine allows $3,000 of the capital loss to reduce NY AGI and gives no wage deduction for the $21,208 employer-paid insurance premiums."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of about $106,394 shows it left out the $1,000 dependent exemption. Its $5,815 also exceeds what the 2026 HOH schedule (5.4% from $20,900 to $107,650) plus the $156.51 supplemental tax yields."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Applied the $11,200 standard deduction with no dependent exemption and reported $5,977.08, which is $378 too high. That fits taxing part of the HOH 5.4% band ($20,900–$107,650) at a higher 5.9–6% rate instead of using the 2026 HOH schedule ($5,442.63) plus the $156.51 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Itemized about $23,600 to reach $92,961.65 of taxable income, but allowable NY itemized deductions are only $9,328.22, so the $11,200 HOH standard deduction applies. The excess fits counting the $21,208 pre-tax employer insurance premiums as deductible medical expenses."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"Applied the 2026 HOH brackets after the standard deduction and dependent exemption, but its $5,458 is only $15 above the $5,442.63 main tax. It left out nearly all of the $156.51 supplemental recapture tax, which applies because NY AGI is above $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Subtracted a $120 NY child and dependent care credit as nonrefundable, but the NY CDCC is refundable and excluded from tax before refundable credits. No nonrefundable credit applies, so the answer is the $5,442.63 main tax plus the $156.51 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Itemized about $23,600 to reach $92,961.65 of taxable income, but allowable NY itemized deductions are only $9,328.22, so the $11,200 HOH standard deduction applies. The excess fits treating the $21,208 employer insurance premiums as deductible medical expenses."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"Applied only the standard deduction without the $1,000 dependent exemption and reported $5,731.29, which is $132 too high. That exceeds what the 2026 HOH schedule (3.9%/4.4%/5.15%/5.4% through $107,650) on $105,393.62 plus the $156.51 supplemental tax produces."
-us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Gave no computation. The $5,500 is $99 below the reference, which fits stopping near the bracket tax on $105,393.62 and leaving out the $156.51 supplemental recapture tax triggered by NY AGI above $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its NY AGI of $96,454 wrongly takes the $21,208 employer insurance premiums out of wages; federal AGI is $117,661.12. It used a $12,100 standard deduction instead of $11,200. It also subtracted a $120 NY CDCC, which is refundable and excluded from this measure."
-us,scenario_082,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Got taxable income of $105,394, but used the pre-2026 formula ($901 plus 5.5% above $20,900) instead of the 2026 rates (3.9%/4.4%/5.15%/5.4%), which give $5,442.63. It also left out the $156.51 supplemental recapture tax for NY AGI above $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"Itemized about $23,000 to reach $93,545 of taxable income, but allowable NY itemized deductions are only $9,328.22, so the $11,200 HOH standard deduction applies. It also used the single schedule's 6% rate above $80,650 instead of the HOH 5.4% band through $107,650."
-us,scenario_082,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Left out the $1,000 dependent exemption ($106,386 of taxable income), used wrong HOH bracket widths ($5,400 and $3,000 instead of $4,850 and $3,250) and the pre-2026 rates, and skipped the supplemental tax. These errors happen to nearly cancel, landing $1.14 below the correct total of $5,442.63 main plus $156.51 supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Subtracted $2,000 of exemptions when NY allows only the $1,000 dependent exemption, and did not subtract the $68 SALT refund from NY AGI. It then deducted a $120 NY child care credit, which is refundable and excluded from tax before refundable credits."
-us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value for state_income_tax_before_refundable_credits, so there is no answer to score against $5,599.14."
-us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"Got taxable income of $105,394, but taxed it 'through the 5.75% bracket' instead of at the 2026 HOH rate of 5.4% on $20,900–$107,650, which gives $5,442.63. It reported no supplemental tax, leaving out the $156.51 recapture tax."
-us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"Assumed the $33,000 capital loss wiped out most of the income, but net capital losses offset ordinary income only up to $3,000, so NY AGI is $117,593.62. It also credited a child tax credit that doesn't reduce NY tax before refundable credits, which led to a $0 liability."
-us,scenario_082,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"Approximated with rough '4–6%' rates instead of the 2026 HOH schedule (5.4% on $20,900–$107,650, giving $5,442.63) plus the separately computed $156.51 supplemental tax, which overshot by $101."
-us,scenario_082,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Its taxable income of about $85,200 is about $20,200 below the correct $105,393.62, which fits wrongly subtracting the $21,208 employer insurance premiums from wages. It also subtracted a $120 NY child care credit, which is refundable and excluded from this measure."
-us,scenario_082,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"Used an $8,000 standard deduction for head of household instead of $11,200, and applied old 5.85% and 6.25% rates instead of the 2026 HOH 5.4% band through $107,650. It also subtracted a $120 NY CDCC, which is refundable and excluded from tax before refundable credits."
-us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Deducted the full $30,711 net capital loss, giving federal AGI of $89,951, instead of the $3,000 limit, which gives $117,661.12. It filed as single with the $8,000 deduction instead of HOH with $11,200. It then raised its $4,292.89 tax to $4,755.44 'after household credit', which contradicts its own arithmetic."
-us,scenario_082,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Overstated federal AGI at $139,586 (correct: $117,661.12), added a made-up 'federal deduction adjustment', and used the $8,000 single deduction instead of the $11,200 HOH amount. It then subtracted the refundable Empire State Child Credit, a nonexistent NY CTC, and a household credit that is phased out at this income."
-us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Used the pre-2025 ESCC formula (33% of a $1,000 CTC = $330), not the enhanced $1,000 under-4 credit with a $16.50 per $1,000 phase-out above $75,000, which yields $307. Guessed about $150 for the care credit instead of the 2026 606(c-2) credit of about 29.352% × $3,000 = $880.55."
-us,scenario_082,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Computed the care credit as 20% of the $600 federal CDCC ($120), missing that 606(c-2) from 2026 applies about 29.352% to the $3,000 of qualifying expenses ($880.55). The ESCC phase-out also used the exact $42,654 excess at 1.65%, not $16.50 per full $1,000 (42 × $16.50 = $693), giving $296.21 instead of $307."
-us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly stated that New York has no refundable child credit. The Empire State Child Credit gives $1,000 for a child under 4, phased down to $307 at this AGI, and the 2026 606(c-2) care credit is a refundable $880.55. Both were omitted."
-us,scenario_082,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Assumed income eliminates all refundable NY credits. The ESCC ($1,000 less a $693 phase-out = $307) and the 2026 606(c-2) child care credit ($880.55 on $3,000 of expenses) both remain at about $117,600 AGI."
-us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Invented a $110 under-4 ESCC amount instead of the $1,000 base less $16.50 per $1,000 above $75,000 ($307). It also wrongly treated the NY care credit as nonrefundable above about $50K, when the 2026 606(c-2) credit is a refundable $880.55."
-us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Asserted that the ESCC fully phases out and that the care credit is nonrefundable at this AGI. The $16.50 per $1,000 phase-out only removes $693 of the $1,000 under-4 credit, and the 2026 606(c-2) care credit is a refundable $880.55."
-us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Treated income as too high for any refundable NY credit. At about $117,600 AGI, the ESCC still pays $307 and the 2026 606(c-2) care credit pays $880.55."
-us,scenario_082,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Used the old care-credit rule (20% of the $600 federal CDCC = $120) instead of the 2026 606(c-2) credit, which applies about 29.352% to $3,000 of expenses ($880.55). It also counted 43 thousands of excess instead of 42 full thousands, giving an ESCC of $290.50 instead of $307."
-us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Applied the superseded ESCC formula of 33% of a $2,000 federal CTC ($660) with no phase-out. The 2025+ under-4 credit is $1,000 less $16.50 per $1,000 above $75,000, or $307. It also used the old 20%-of-federal care credit ($120) instead of the 2026 606(c-2) amount of $880.55."
-us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,Concluded the ESCC is phased out and ignored the child care credit entirely. It also understated AGI by subtracting retirement deferrals without counting the dividends correctly. The $307 ESCC and the refundable $880.55 606(c-2) care credit both apply.
-us,scenario_082,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Assumed the ESCC is fully phased out, when the $1,000 under-4 credit loses only $693 and leaves $307. It also computed the care credit as 20% of the federal $600 ($120) instead of the 2026 606(c-2) credit of about 29.352% × $3,000 = $880.55."
-us,scenario_082,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Considered only the NY EITC and ignored the Empire State Child Credit ($307 after phase-out) and the refundable 2026 606(c-2) child care credit ($880.55).
-us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Omitted the $307 ESCC for the child under 4. It also computed the care credit under the old 20%-of-federal-CDCC rule ($120) instead of the 2026 606(c-2) percentage of $3,000 in expenses ($880.55)."
-us,scenario_082,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Applied the pre-2023 rule that excludes children under 4 from the ESCC. The enhanced credit gives $1,000 for a child under 4, phased to $307. It also omitted the refundable 2026 606(c-2) care credit of $880.55."
-us,scenario_082,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Gave the full $1,000 ESCC without the $16.50 per $1,000 phase-out above $75,000 (−$693, leaving $307). It also used the old 20%-of-federal care credit ($120) instead of the 2026 606(c-2) amount of $880.55."
-us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used the old 33%-of-CTC ESCC ($330) instead of $1,000 less $693 ($307), and guessed $300 for the care credit instead of the 606(c-2) amount of $880.55. It also added an NYC school tax credit, but the household has no NYC residence."
-us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Used the superseded 33%-of-$1,000 ESCC ($330) instead of the enhanced $1,000 under-4 credit phased down to $307. It also omitted the refundable 2026 606(c-2) child care credit of $880.55."
-us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Applied the old $100 ESCC minimum instead of the 2025+ $1,000 under-4 base less a $693 phase-out ($307). It also used 20% of the federal CDCC ($120) instead of the 2026 606(c-2) credit of $880.55."
-us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Applied the low-income 110%-of-federal care credit rate ($660) at about $117,600 AGI. The 2026 606(c-2) rate at this income is about 29.352% of $3,000, or $880.55. It also used the old 33% ESCC ($330) instead of $307."
-us,scenario_082,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Returned zero and missed both refundable NY credits: the ESCC of $1,000 less a $693 phase-out ($307) and the 2026 606(c-2) child care credit ($880.55)."
-us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Used the old 33%-of-$1,000 ESCC ($330) instead of the phased enhanced credit ($307). It also omitted the refundable 2026 606(c-2) care credit of $880.55."
-us,scenario_082,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Counted only an ESCC of $330 under the superseded 33% formula instead of $307 under the $1,000 base with a $16.50 per $1,000 phase-out. It also omitted the $880.55 606(c-2) child care credit."
-us,scenario_082,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Treated the ESCC as phased out, when $307 remains. It also computed the care credit as 20% of the federal $600 ($120) instead of the 2026 606(c-2) percentage of $3,000 in expenses ($880.55)."
-us,scenario_082,state_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"Dropped the ESCC under the old age-4+ rule, but the enhanced credit covers a child under 4 at $1,000 less $693 ($307). It also used the old 20%-of-federal care credit ($120) instead of the 2026 606(c-2) amount of $880.55."
-us,scenario_082,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,No answer or reasoning was returned for state_refundable_credits. The correct total is the $307 ESCC plus the $880.55 606(c-2) care credit.
-us,scenario_082,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Treated the enhanced ESCC as 100% of the $2,200 federal CTC. The 2025+ credit is a flat $1,000 for a child under 4, reduced by $16.50 per $1,000 of AGI above $75,000, which leaves $307. It also omitted the $880.55 606(c-2) care credit."
-us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,Found no refundable NY credits and omitted the $307 ESCC for the child under 4 and the refundable 2026 606(c-2) child care credit of $880.55.
-us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Considered only the NY EITC and omitted the ESCC ($1,000 less a $693 phase-out = $307) and the refundable 606(c-2) child care credit ($880.55)."
-us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Used the old care-credit rule (20% of the $600 federal CDCC = $120) instead of the 2026 606(c-2) credit of about 29.352% × $3,000 = $880.55. It also wrongly treated the ESCC as fully phased out, when $307 remains."
-us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Gave the full $1,000 under-4 ESCC without its $16.50 per $1,000 phase-out above $75,000, which removes $693 and leaves $307. It also omitted the $880.55 606(c-2) care credit."
-us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Used the old 20%-of-federal care credit ($120) instead of the 2026 606(c-2) credit on $3,000 of expenses ($880.55). It also counted 43 rather than 42 full thousands of excess AGI, giving an ESCC of $290.50 instead of $307."
-us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Applied the pre-2023 age-4 floor to the ESCC. The enhanced credit pays $1,000 for a child under 4, phased to $307. It also wrongly concluded that income bars the refundable 606(c-2) care credit, which is $880.55 here."
-us,scenario_082,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Computed the care credit as 20% of the $600 federal CDCC ($120) instead of the 2026 606(c-2) percentage of $3,000 in expenses ($880.55). It also counted a partial thousand in the phase-out, giving an ESCC of $290.50 instead of $307."
-us,scenario_082,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Said income is too high for the ESCC, but the $16.50 per $1,000 phase-out leaves $307 of the $1,000 under-4 credit. It also used the old 20%-of-federal care credit ($120) instead of $880.55 under 606(c-2)."
-us,scenario_082,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Wrongly treated the NY dependent care credit as nonrefundable and the child credit as unavailable. The refundable 2026 606(c-2) care credit is $880.55, and the ESCC is $307 after phase-out."
-us,scenario_082,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Got close on the ESCC ($312 vs $307) but used the superseded 20%-of-federal care credit ($120). Under the 2026 606(c-2) credit, about 29.352% applies to $3,000 of expenses, giving $880.55."
-us,scenario_082,state_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,Included only the old 20%-of-federal care credit ($120). It omitted the $307 ESCC and missed that the 2026 606(c-2) care credit is $880.55.
-us,scenario_082,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,Found no qualifying credits and missed the $307 ESCC for the child under 4 and the refundable $880.55 606(c-2) child care credit.
-us,scenario_082,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Claimed the federal CTC is phased out, but at about $117,600 AGI head of household it is not. The enhanced ESCC is a $1,000 under-4 base with its own $16.50 per $1,000 phase-out, which leaves $307. It also omitted the $880.55 606(c-2) care credit."
-us,scenario_082,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Used the old $100 ESCC minimum on the theory that the federal CTC is phased out, instead of the 2025+ $1,000 under-4 credit less $693 ($307). It also used 20% of the federal CDCC ($120) instead of the 2026 606(c-2) credit of $880.55."
-us,scenario_082,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Set the under-4 ESCC at $660 (33% of a $2,000 CTC, no phase-out). The enhanced credit is $1,000 less $16.50 per $1,000 above $75,000, or $307. The care credit was taken as 20% of the federal $600 ($120) instead of the 606(c-2) amount of $880.55."
-us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,Considered only the state EITC and omitted the $307 ESCC and the refundable $880.55 606(c-2) child care credit.
-us,scenario_082,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Treated the NY child care credit as absorbed by tax liability, but the 2026 606(c-2) credit is refundable at $880.55. It also omitted the $307 ESCC."
-us,scenario_082,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,No answer or reasoning was returned for state_refundable_credits. The correct total is the $307 ESCC plus the $880.55 606(c-2) care credit.
-us,scenario_082,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Used the pre-2025 ESCC structure (recomputed 2017-law CTC with a $100 minimum) instead of the enhanced $1,000 under-4 credit less a $693 phase-out ($307). It also used the old 20%-of-federal care credit ($120) instead of the 2026 606(c-2) amount of $880.55."
-us,scenario_082,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Concluded that income excludes the ESCC, but the $16.50 per $1,000 phase-out leaves $307. It also ignored the refundable $880.55 606(c-2) child care credit."
-us,scenario_082,state_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,Looked only at the NY EITC and omitted the $307 ESCC and the refundable $880.55 606(c-2) child care credit.
-us,scenario_082,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,Treated income as barring all refundable NY credits. The ESCC still pays $307 and the 2026 606(c-2) care credit pays $880.55 at this AGI.
-us,scenario_082,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Applied a 5% phase-out rate to the ESCC. The actual reduction is $16.50 per full $1,000 above $75,000 (42 × $16.50 = $693), which leaves $307. It also omitted the $880.55 606(c-2) child care credit."
-us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,Checked only the NY EITC and missed the $307 ESCC for the child under 4 and the refundable $880.55 606(c-2) child care credit.
-us,scenario_082,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Got roughly the ESCC ($313 vs $307) but left out the refundable 2026 606(c-2) child and dependent care credit, which is about 29.352% of $3,000, or $880.55."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It overstated gross NY head-of-household tax at ~$5,891 when the 2026 schedule on $105,394 gives $5,442.63, plus $156.51 supplemental tax, for $5,599.14. It then subtracted a ~$396 NY child and dependent care credit, but that credit is refundable in New York and does not belong in this figure. It also kept the $68 refund in NY AGI instead of subtracting it."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It computed the supplemental tax (recapture) using a 5.4% flat-rate tax. The recapture compares regular tax to a flat 5.9% tax, the rate for income above $107,650, so the supplemental tax is about $156, not $49.75. It also left the $68 refund in NY AGI and deducted the nondeductible $8 IRA contribution."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"Its NY AGI of $117,654 kept the $68 state refund, which New York subtracts, and deducted an $8 traditional IRA contribution that is not deductible because the head is a workplace-plan participant above the phaseout. Taxable income therefore came out $60 too high at $105,454 instead of $105,393.62. That error, together with its own recapture calculation, left the answer $1.99 above $5,599.14."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used federal taxable income with a $14,600 federal standard deduction (giving $62,149) instead of NY AGI ($117,594) minus the NY head-of-household standard deduction ($11,200) and the $1,000 dependent exemption. Also, its $5,649 does not follow from its own $62,149 base under NY rates."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It guessed $5,200 without applying the 2026 NY head-of-household schedule to $105,394 of taxable income. That schedule gives $5,442.63, and the $156.51 supplemental tax for NY AGI above $107,650 brings the total to $5,599.14."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It filed as single with an $8,000 standard deduction and no dependent exemption. A parent with a qualifying 1-year-old files head of household, which gives an $11,200 deduction plus a $1,000 exemption. It then took its $6,221 down to $5,076 with an unexplained reduction, even though no nonrefundable credit applies."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It used single status with an $8,000 deduction and no $1,000 dependent exemption, and it did not subtract the $181 traditional 401(k) contribution from wages. After computing $6,326 under the single rates, it applied an arbitrary ~$1,050 'modest adjustment' although no nonrefundable credit is available."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It applied an $8,000 standard deduction to a head-of-household filer instead of $11,200 and left out the $1,000 dependent exemption, so taxable income came out at ~$109,654 instead of $105,394. It then rounded to $6,000 using rough 5.5–6% rates instead of the 2026 schedule plus the supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Its 2026 head-of-household schedule produced $5,808.15 on $105,386. The reduced rates of 3.9%/4.4%/5.15%/5.4% at the $12,800/$17,650/$20,900 thresholds give about $5,442 on that income. It also understated the recapture at $81.40; the flat 5.9% comparison yields about $156."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It filed as single with outdated single brackets (5.85%) and dropped the $1,000 dependent exemption. Its $9,917 itemized total ignored the charitable floor (NY itemized is $9,328, below the $11,200 head-of-household standard deduction). It then subtracted a $120 NY child care credit, which is refundable and excluded from this figure."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It used single status and $9,917 of itemized deductions instead of the head-of-household $11,200 standard deduction plus the $1,000 dependent exemption. After computing $6,193, it dropped to $4,800 by citing a household credit, but that credit is fully phased out at this income."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It reached the correct taxable income of $105,394 but estimated the tax roughly at $5,400. The 2026 head-of-household schedule gives $5,442.63, and it left out the $156.51 supplemental tax that applies because NY AGI exceeds $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It understated AGI at ~$86,949 when the $3,000 capital-loss cap leaves federal AGI at $117,661. It also took $2,000 of exemptions; only the child earns the $1,000 exemption. Finally, it subtracted the Empire State Child Credit, which is refundable and excluded from this figure."
+us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It claimed $20,691 of itemized deductions, but NY itemized deductions total only $9,328, so the $11,200 head-of-household standard deduction applies. It also used the 2025 rates with a wrong $13,580 first-bracket width and left out the 2026 rate cut and the supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It took the $11,200 standard deduction but not the $1,000 dependent exemption, which inflated pre-credit tax to $5,731.77 instead of $5,599.14. It then subtracted a $120 NY child and dependent care credit, which is refundable and does not reduce tax before refundable credits."
+us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It took $2,000 of exemptions, but New York gives an exemption only for dependents ($1,000 for the child), so its taxable income of $104,386 was $1,000 too low. It also left out the $156.51 supplemental tax that applies because NY AGI exceeds $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It used a $10,500 standard deduction instead of the $11,200 head-of-household amount and left out the $1,000 dependent exemption. It subtracted a $30 household credit, which is fully phased out at this income, and it never added the supplemental recapture tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $4,380 matches taxable income of only about $84,000, which means it dropped roughly $21,000 of income or deductions from the base. The correct base is NY AGI of $117,594 minus $11,200 and $1,000, or $105,394. The 2026 head-of-household rates on that base plus the $156.51 supplemental tax give $5,599.14."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its taxable income of $84,178 is $105,394 minus the $21,208 employer health premiums (and $8 IRA). It treated the premiums as a further reduction of the listed wages, but they do not reduce taxable wages here. It also used the 2025 rates (5.5%) instead of the reduced 2026 rates, and because its AGI fell below $107,650 it missed the supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its NY AGI of $96,454 equals federal AGI minus the $21,208 employer insurance premiums, which do not reduce the listed wages; correct NY AGI is $117,594. It also subtracted $17,200 instead of $12,200 for the standard deduction plus exemption, and so never triggered the supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"With no derivation, it gave $6,040.50, which overstates the tax by about $440. The correct path is $105,394 taxable under the head-of-household schedule: $5,442.63 plus $156.51 supplemental tax. An answer that high is what single-filer brackets or a missing head-of-household deduction or exemption produces."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $4,220 matches taxable income of about $82,000–$84,000, which means it removed roughly $21,000, the size of the employer insurance premiums, from a base that should be $105,394. It also missed the $156.51 supplemental tax that applies when NY AGI exceeds $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $3,591 matches taxable income of roughly $70,000, far below the correct $105,394 (NY AGI of $117,594 minus the $11,200 standard deduction and $1,000 exemption). It also left out the $156.51 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the right $11,200 deduction and $1,000 exemption but never added the supplemental tax. The 2026 head-of-household schedule gives $5,442.63, and the $156.51 recapture for NY AGI above $107,650 brings the total to $5,599.14, which $5,533 falls short of."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It subtracted the $68 refund twice, giving NY AGI of $117,518, and it subtracted 'household/child care' credits. The household credit is fully phased out and the NY child care credit is refundable, so neither reduces tax before refundable credits. It also never added the supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"It returned no usable value or explanation for state_income_tax_before_refundable_credits, so nothing could be scored against the $5,599.14 result."
+us,scenario_082,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It inflated NY itemized deductions to $27,966. It counted ~$4,700 of NY income tax (not deductible on the NY return) and ~$13,349 of medical expenses (built by treating the $21,208 employer premiums as medical expenses). Actual NY itemized deductions are $9,328, below the $11,200 standard deduction. It also subtracted the refundable $120 NY child care credit."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave a rough $5,150 without applying the 2026 head-of-household schedule to $105,394 of taxable income ($5,442.63). It also omitted the $156.51 supplemental tax that applies because NY AGI exceeds $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $7,777 estimate overstates the tax by about $2,200. The correct figure is $105,394 taxable under the 2026 head-of-household rates: $5,442.63 plus $156.51 supplemental tax. Itemizing gives no benefit here, because NY itemized deductions of $9,328 are below the $11,200 standard deduction."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"Its $5,548 is $901 plus 5.5% of income above $20,900, which is the 2025 head-of-household schedule. The 2026 rates were cut to 3.9%/4.4%/5.15%/5.4%, giving $5,442.63. It also left out the $156.51 supplemental tax for NY AGI above $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $2,954.80 is roughly half the correct liability. Taxable income of $105,394 under the 2026 head-of-household rates gives $5,442.63, plus $156.51 supplemental tax. An answer that low would require taxable income near $55,000, so it dropped most of the income base or applied rates far below New York's 3.9–5.4%."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Its taxable income of $106,394 leaves out the $1,000 dependent exemption for the child. Its graduated tax of $5,815 also overstates the 2026 head-of-household result, which is $5,442.63 plus $156.51 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It applied only the $11,200 standard deduction, leaving out the $1,000 dependent exemption. Its $5,977.08 also overstates the 2026 head-of-household schedule, which gives $5,442.63 on $105,394 plus the $156.51 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It itemized about $23,600 to reach taxable income of $92,961.65, which only works by counting the $21,208 employer premiums as medical expenses. NY itemized deductions are $9,328, so the $11,200 standard deduction applies and taxable income is $105,394."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It had the right deduction and exemption structure but stopped at the main-schedule tax. It never added the $156.51 supplemental tax that applies because NY AGI of $117,594 exceeds $107,650, so the total should be $5,442.63 + $156.51 = $5,599.14."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,other,False,"It subtracted a $120 dependent-care credit, but New York's child and dependent care credit is refundable and excluded from tax before refundable credits. Its pre-credit tax of $5,665.24 also overstates the correct $5,599.14."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It itemized about $23,600 to reach taxable income of $92,961.65, which only works by counting the $21,208 employer premiums as medical expenses. NY itemized deductions are $9,328, so the $11,200 standard deduction applies and taxable income is $105,394."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It applied only the standard deduction, leaving out the $1,000 dependent exemption, and computed $5,731.29. The 2026 head-of-household schedule on $105,394 gives $5,442.63, plus the $156.51 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It rounded to $5,500 without computing the 2026 head-of-household schedule ($5,442.63 on $105,394 of taxable income) or adding the $156.51 supplemental tax for NY AGI above $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Its NY AGI of $96,454 removes the $21,208 employer insurance premiums from wages, but those premiums do not reduce the listed wages; correct NY AGI is $117,594. It used a $12,100 standard deduction instead of $11,200, and it subtracted a $120 NY child care credit, which is refundable."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied the 2025 head-of-household formula ($901 + 5.5% over $20,900) instead of the 2026 reduced rates (3.9%/4.4%/5.15%/5.4%), which give $5,442.63. It also left out the $156.51 supplemental tax for NY AGI above $107,650."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It itemized about $23,000, but NY itemized deductions are only $9,328, so the $11,200 head-of-household standard deduction applies. It also used the single schedule ($4,271 + 6% over $80,650) instead of the head-of-household rates."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It left out the $1,000 dependent exemption (taxable $106,386 instead of $105,394), used 2025 rates with wrong bracket widths ($5,400 and $3,000 instead of $4,850 and $3,250), and dropped the $156.51 supplemental tax. These errors happen to offset and land $1.14 below $5,599.14."
+us,scenario_082,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It took $2,000 of exemptions, but New York allows the $1,000 exemption only for the dependent child. It also subtracted a $120 NY child and dependent care credit, which is refundable and excluded from tax before refundable credits."
+us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no usable value or explanation for state_income_tax_before_refundable_credits, so nothing could be scored against the $5,599.14 result."
+us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It had the correct taxable income of $105,394 but applied a nonexistent 5.75% bracket. The 2026 head-of-household rate up to $107,650 is 5.4%, giving $5,442.63. It also never added the $156.51 supplemental recapture tax."
+us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It let the $30,711 net capital loss offset income in full. The deduction is capped at $3,000, leaving NY AGI at $117,594. With $105,394 of taxable income the tax is $5,599.14, not zero."
+us,scenario_082,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It rounded to $5,700 using approximate 4–6% rates instead of the reduced 2026 head-of-household rates (3.9%/4.4%/5.15%/5.4%), which give $5,442.63 on $105,394 before the $156.51 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Its taxable income of about $85,200 is roughly $20,000 too low, consistent with treating the $21,208 employer premiums as a wage exclusion; correct taxable income is $105,394. It also subtracted a $120 NY child care credit, which is refundable."
+us,scenario_082,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used an $8,000 standard deduction for head of household instead of $11,200, and outdated 5.85%/6.25% brackets instead of the 2026 rates of 3.9%/4.4%/5.15%/5.4%/5.9%. It also subtracted the refundable $120 NY child care credit."
+us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It netted the full $30,711 capital loss against income instead of capping the deduction at $3,000, which understated AGI at $89,951 instead of $117,661. It also filed single with an $8,000 deduction instead of head of household with $11,200."
+us,scenario_082,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It overstated AGI at $139,586 when the capital-loss cap leaves $117,661, and it used single-status figures ($8,000) plus an invented 'federal deduction adjustment.' It also subtracted the Empire State Child Credit and a household credit; the first is refundable and the second is fully phased out."
+us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Used the old Empire State Child Credit formula (33% of a $1,000 pre-TCJA credit = $330) instead of the 2026 enhanced $1,000 under-4 credit reduced by $693 to $307. Guessed about $150 for the care credit instead of applying the 606(c-2) rate of about 29.35% to $3,000 of expenses, which gives $880.55."
+us,scenario_082,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Correctly started the child credit from the $1,000 under-4 base but used a continuous 1.65% phase-out, getting $296.21 instead of the $693 reduction that leaves $307. Its main error was the care credit: it used the old 20%-of-federal-CDCC rule ($120) instead of the 2026 606(c-2) formula, which applies about 29.35% to $3,000 of expenses for $880.55."
+us,scenario_082,state_refundable_credits,claude-fable-5.2,llm_error,state_local_rule,False,"Phased out the $1,000 under-4 child credit over 43 $1,000 steps ($709.50) instead of the $693 reduction that leaves $307. Also used the old care-credit rule of 20% of the $600 federal credit ($120) instead of the 2026 606(c-2) rate of about 29.35% on $3,000 of expenses ($880.55)."
+us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly said New York has no refundable child credit, missing the refundable Empire State Child Credit ($307 after phase-out for the 1-year-old). Also missed the refundable 2026 child and dependent care credit under 606(c-2), worth $880.55."
+us,scenario_082,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Said the income was too high for any refundable credit. It is not: the $1,000 under-4 child credit is only partly phased out at about $117.7k federal AGI, leaving $307. The 2026 606(c-2) care credit adds $880.55 on $3,000 of expenses."
+us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Made up a $110 under-4 child credit amount instead of using the enhanced $1,000 base less the $693 phase-out ($307). Wrongly treated the New York care credit as nonrefundable at this income and left out the $880.55 refundable 606(c-2) credit."
+us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Assumed the child credit was fully phased out, but $1,000 less a $693 reduction leaves $307. Also called the care credit nonrefundable at higher AGI, when the 2026 606(c-2) care credit is refundable and worth $880.55."
+us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Said the income was too high for the child credit and the care credit. At about $117.7k federal AGI, the $1,000 under-4 child credit still leaves $307, and the 606(c-2) care credit gives $880.55 refundable."
+us,scenario_082,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Used 43 phase-out steps of $16.50 ($709.50) on the $1,000 under-4 child credit instead of the $693 reduction that leaves $307. Applied the outdated 20%-of-federal-CDCC care credit ($120) instead of the 2026 606(c-2) formula, which applies about 29.35% to $3,000 of expenses ($880.55)."
+us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Used the old 33%-of-federal-CTC child credit formula ($660) and said there was no phase-out, instead of the enhanced $1,000 under-4 credit reduced by $693 to $307. Applied the old 20%-of-federal care credit ($120) instead of the 606(c-2) $880.55, and also understated AGI by subtracting the full $33,000 long-term loss."
+us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Assumed the child credit was eliminated, but the $1,000 under-4 credit loses only $693 at this federal AGI, leaving $307. Never considered the refundable 2026 606(c-2) care credit of $880.55."
+us,scenario_082,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Assumed the child credit was fully phased out instead of leaving $307 ($1,000 under-4 base less $693). Used the outdated 20%-of-$600 federal care credit ($120) instead of the 606(c-2) applicable percentage on $3,000 of expenses ($880.55)."
+us,scenario_082,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Considered only the New York EITC. Missed the refundable Empire State Child Credit ($307 for the under-4 child after phase-out) and the refundable 2026 606(c-2) care credit ($880.55).
+us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Left out the $307 Empire State Child Credit ($1,000 under-4 base less $693). Calculated the care credit under the old 20%-of-federal rule ($120) instead of the 606(c-2) rate of about 29.35% on $3,000 ($880.55)."
+us,scenario_082,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Used the outdated rule that children under 4 are excluded from the Empire State Child Credit. Under 2026 law they get a $1,000 base, which phases down to $307 here. Also left out the refundable 606(c-2) care credit of $880.55."
+us,scenario_082,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Claimed the full $1,000 under-4 child credit without the $693 income phase-out above the $75,000 head-of-household threshold, which leaves $307. Used the old 20%-of-federal care credit ($120) instead of the 606(c-2) $880.55."
+us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used the old $330 child credit instead of the enhanced $1,000 under-4 credit reduced to $307. Guessed $300 for the care credit instead of the 606(c-2) $880.55, and added an NYC school tax credit even though the household is not in NYC."
+us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Used the old 33% child credit amount ($330) instead of the $1,000 under-4 credit less the $693 phase-out ($307). Left out the refundable 606(c-2) care credit of $880.55 entirely."
+us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Used the $100 per-child minimum instead of the enhanced $1,000 under-4 child credit, which phases down to $307. Used the old 20%-of-federal care credit ($120) instead of the 2026 606(c-2) calculation ($880.55)."
+us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Applied the lowest-income 110% care-credit rate to the federal credit ($660), which does not fit this AGI, instead of the 606(c-2) applicable percentage of about 29.35% on $3,000 ($880.55). Used the old $330 child credit instead of $307 ($1,000 under-4 base less $693)."
+us,scenario_082,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Reported no refundable credits. It missed the $307 child credit ($1,000 under-4 base less $693 phase-out) and the $880.55 refundable care credit under 606(c-2)."
+us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Used the old $330 child credit instead of the enhanced $1,000 under-4 credit reduced to $307. Left out the refundable 606(c-2) care credit of $880.55."
+us,scenario_082,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Reported only an old-formula $330 child credit instead of $307 under the enhanced $1,000 under-4 credit with its $693 phase-out. Left out the $880.55 refundable care credit under 606(c-2)."
+us,scenario_082,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Assumed the child credit was phased out, but $307 remains of the $1,000 under-4 base. Used the old 20%-of-$600 federal care credit ($120) instead of the 2026 606(c-2) rate of about 29.35% on $3,000 ($880.55)."
+us,scenario_082,state_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"Applied the outdated age-4-and-older rule and gave the 1-year-old no child credit, but 2026 law provides a $1,000 under-4 base that phases down to $307. Used the old 20%-of-federal care credit ($120) instead of the 606(c-2) $880.55."
+us,scenario_082,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"No parseable value was returned for state_refundable_credits, so nothing could be scored against the $1,187.55 total (a $307 child credit plus an $880.55 care credit)."
+us,scenario_082,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Described the enhanced child credit as 100% of the federal CTC ($2,200). It is actually a $1,000 under-4 base, reduced by $693 at this federal AGI to $307. Also left out the $880.55 refundable care credit under 606(c-2)."
+us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no refundable New York credits. It missed the $307 child credit for the 1-year-old and the $880.55 refundable care credit on the $9,600 of child care expenses."
+us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Looked only for EITC-type credits. It missed the $307 child credit ($1,000 under-4 base less $693) and the 606(c-2) care credit ($880.55) that the 1-year-old and the child care expenses produce."
+us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Said no child credit remained, but $307 is left after the $693 phase-out of the $1,000 under-4 base. Used the old rule of 20% of the $600 federal credit ($120) instead of the 2026 606(c-2) percentage applied to $3,000 of expenses ($880.55)."
+us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Claimed the full $1,000 under-4 child credit without the $693 phase-out above $75,000 federal AGI, which leaves $307. Left out the $880.55 refundable care credit under 606(c-2)."
+us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Phased out the $1,000 child credit to $290.50 (43 steps) instead of $307 (a $693 reduction). Used the old 20%-of-federal care credit ($120) instead of the 606(c-2) rate of about 29.35% on $3,000 ($880.55)."
+us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Wrongly treated the 1-year-old as outside the child credit's age range. The 2026 credit gives children under 4 a $1,000 base, which phases down to $307. Also wrongly denied the refundable 606(c-2) care credit of $880.55."
+us,scenario_082,state_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Phased out the child credit to $290.50 instead of the reference's $307 ($1,000 less $693). Used the old 20%-of-federal care credit ($120) instead of the 2026 606(c-2) calculation on $3,000 of expenses ($880.55)."
+us,scenario_082,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Said income was too high for any child credit, but $307 of the $1,000 under-4 base survives the phase-out. Used the outdated 20%-of-$600 care credit ($120) instead of the 606(c-2) $880.55."
+us,scenario_082,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,Wrongly treated the New York care credit as nonrefundable and the child credit as unavailable. Both are refundable here: $307 for the child credit and $880.55 for the care credit under 606(c-2).
+us,scenario_082,state_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Its $312 child credit was close to $307, but its main error was the care credit. It used the old 20%-of-federal rule ($120) instead of the 2026 606(c-2) applicable percentage of about 29.35% on $3,000 of expenses ($880.55)."
+us,scenario_082,state_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,"Used the old care-credit rule of 20% of the federal credit ($120) instead of the 606(c-2) $880.55. Left out the $307 child credit ($1,000 under-4 base less $693)."
+us,scenario_082,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,Found no refundable credits. It missed the $307 child credit for the under-4 child and the $880.55 refundable 2026 care credit under 606(c-2).
+us,scenario_082,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Wrongly said the federal CTC is fully phased out at about $117.7k head-of-household AGI (the federal phase-out starts at $200,000) and tied the state child credit to it. The 2026 credit is a $1,000 under-4 base less $693, which is $307. Also left out the $880.55 care credit."
+us,scenario_082,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Wrongly treated the federal CTC as phased to zero and fell back to the $100 minimum. The child credit is actually $1,000 less $693, which is $307. Used the old 20%-of-federal care credit ($120) instead of the 606(c-2) $880.55."
+us,scenario_082,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Used a $660 child credit (33% of a $2,000 federal CTC) instead of the enhanced $1,000 under-4 base reduced by $693 to $307. Used the old 20%-of-federal care credit ($120) instead of the 606(c-2) $880.55."
+us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,Considered only the state EITC. It missed the refundable $307 child credit and the refundable $880.55 care credit under 606(c-2).
+us,scenario_082,state_refundable_credits,inkling,llm_error,state_local_rule,False,Wrongly treated the New York care credit as absorbed by tax liability. The 2026 606(c-2) care credit is refundable and worth $880.55 here. Also left out the refundable $307 child credit.
+us,scenario_082,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No parseable value was returned for state_refundable_credits, so nothing could be scored against the $1,187.55 total (a $307 child credit plus an $880.55 care credit)."
+us,scenario_082,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Tied the child credit to a recalculated 2017-law federal CTC and fell back to the $100 minimum, instead of using the 2026 $1,000 under-4 base less the $693 phase-out ($307). Used the old 20%-of-federal care credit ($120) instead of the 606(c-2) $880.55."
+us,scenario_082,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Said the income was outside the child credit's range, but the $1,000 under-4 credit phases down only to $307 at this federal AGI. Also left out the refundable $880.55 care credit under 606(c-2)."
+us,scenario_082,state_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"Focused only on the EITC. It missed the $307 child credit for the 1-year-old and the $880.55 refundable care credit on $3,000 of qualifying expenses."
+us,scenario_082,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,Considered only the state EITC. It missed the refundable child credit ($307) and the refundable 2026 606(c-2) care credit ($880.55).
+us,scenario_082,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Phased out the $1,000 under-4 child credit at 5% of AGI over $75,000. The actual reduction is $16.50 per $1,000 of excess federal AGI, which removes $693 and leaves $307. Also left out the $880.55 refundable care credit under 606(c-2)."
+us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Looked only at the EITC. It missed the $307 child credit ($1,000 under-4 base less $693) and the $880.55 refundable 2026 care credit."
+us,scenario_082,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Its roughly $313 child credit was close to $307, but it left out the refundable child and dependent care credit. Under 606(c-2), that credit applies about 29.35% to $3,000 of expenses for $880.55."
us,scenario_083,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed the individual provisions of the Tax Cuts and Jobs Act expired for 2026 and used an $8,328 single-filer standard deduction. The applicable 2026 standard deduction exceeds $11,010, so taxable income and the resulting 10% bracket tax are both zero."
us,scenario_083,federal_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly determined that the age-20 childless filer fails the EITC age requirement and that no other refundable credit applies, but then submitted $540 instead of carrying the resulting $0 into the output."
us,scenario_083,federal_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly concluded that the filer is too young for the childless EITC and has no qualifying child for refundable CTC, but its $538 output contradicts its own all-zero component calculation."
@@ -7649,8 +7730,9 @@ us,scenario_084,head_medicare_eligible,minimax-m3,prompt_ambiguity,age_disabilit
us,scenario_084,head_medicare_eligible,muse-spark-1.3,prompt_ambiguity,age_disability,False,"The model treated $24,828 of SSDI income plus disability as qualifying for Medicare. The modeled disability pathway requires at least 24 months of SSDI receipt; the facts leave duration unstated, and this output is excluded from scoring."
us,scenario_084,head_medicare_eligible,muse-wonder,prompt_ambiguity,age_disability,False,"The model granted Medicare on disability grounds at age 62 from SSDI receipt alone. The modeled disability pathway requires at least 24 months of SSDI receipt; the facts leave duration unstated, and this output is excluded from scoring."
us,scenario_084,head_medicare_eligible,qwen-3.7-max,prompt_ambiguity,age_disability,False,The model cited the 24-month SSDI rule but treated the constant full-year disability and SSDI status as satisfying it. A status held constant through one tax year does not establish when SSDI receipt began. The output is excluded from scoring because total duration is unstated.
-us,scenario_084,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"Sonnet derived monthly net income of $1,737.59 but never compared it to the SNAP net income limit of 100% FPL (about $1,340/month for one person), which that figure fails by roughly 30%, and it ignored both NC's 200%-FPL BBCE gross cap (about $2,680/month against its own $2,769) and the $56,500 in bank assets against the elderly/disabled resource limit of roughly $4,650. Its own benefit formula returned $0 because 30% of net income ($521.28) exceeds the one-person maximum allotment ($298), after which it discarded that result and substituted an unsupported $183/month figure attributed to a conditional minimum benefit, a rule that provides $23-$24, not $183. It further inflated the excess medical deduction by double-counting the same $3,600 premium (listed once as health insurance premiums excluding Medicare Part B and again as other health insurance premiums) and by including $1,054 of non-prescription over-the-counter costs, which are not deductible medical expenses."
-us,scenario_084,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"DeepSeek asserted the household passes the net income test after the standard and medical deductions, but net income of about $1,738/month exceeds the one-person net income limit near $1,340/month, gross income of $2,769/month exceeds North Carolina's 200%-FPL BBCE gross cap of about $2,680/month, and $56,500 in countable resources exceeds the elderly/disabled asset limit of roughly $4,650. Having wrongly cleared eligibility, it applied the $23/month minimum allotment for one- and two-person households, a floor that attaches only after a household qualifies; an income-ineligible household receives $0, not $276."
+us,scenario_084,snap,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"The model left the $8,400 non-Social Security disability benefit out of SNAP income, which put gross income at about $2,069 a month and under NC's 200% FPL categorical eligibility limit, so it waived the asset test. SNAP counts that benefit as unearned income, so gross income is about $2,769 a month, which is over the roughly $2,608 limit. The household then fails the $1,304 net income test (the model's own net figure was $1,333) and the $4,500 asset test with $56,500 in the bank. The one-person minimum allotment only goes to eligible households, so it does not apply."
+us,scenario_084,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model calculated net income of $1,737.59 a month and found that the standard formula gives $0. It then made up a $183-a-month benefit instead of applying the net income limit of 100% FPL (about $1,304 a month), which its own net income exceeds. It also never applied the $4,500 elderly/disabled asset limit against $56,500 in bank assets, and it never checked the 200% FPL categorical eligibility limit, which the $2,769 monthly gross income also exceeds."
+us,scenario_084,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"The model said the household passes the net income test, but net income after the standard and excess medical deductions is still about $1,333 a month or more, which is above the 100% FPL limit of about $1,304. Gross income of about $2,769 a month rules out NC's 200% FPL categorical eligibility, and the $56,500 in bank assets exceeds the $4,500 elderly/disabled limit, so the household is ineligible. The $23 minimum benefit only goes to eligible households, so it does not apply."
us,scenario_085,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its reasoning reproduced the reference exactly: AGI $45,574, deductions of $24,150 including the $6,000 senior deduction, taxable income $21,424, and tax of about $2,323. It then submitted 940, which contradicts its own computation."
us,scenario_085,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It put the full $27,426 of Social Security into AGI instead of the $14,050.74 taxable portion and used a made-up $27,750 standard deduction. It then wiped out the tax with an elderly credit and a Saver's Credit. Neither applies: the elderly credit is reduced to zero because nontaxable Social Security alone exceeds the $5,000 initial amount, and AGI is above the Saver's Credit limit."
us,scenario_085,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It treated the survivor benefits as 'largely nontaxable'. In fact provisional income of $45,236 exceeds the $34,000 threshold, so $14,050.74 is taxable and AGI is $45,573.74. Its $1,550 understates the $21,423.74 taxable income left after the $24,150 deduction."
@@ -7737,53 +7819,54 @@ us,scenario_086,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_inc
us,scenario_086,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model invented a three-person household despite the prompt listing only the head and child, then conflated the 247% FPL PeachCare/CHIP ceiling with Medicaid eligibility. For the actual two-person household, child1 is at 3.44 times FPL and has no Medicaid eligibility category."
us,scenario_086,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model relied on age-based categorical eligibility without applying Georgia’s income threshold for a 12-year-old child. Child1’s MAGI income level is 3.44 times FPL, above the applicable child Medicaid limits, leaving the Medicaid category as NONE."
us,scenario_086,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_medicaid_eligible, violating the required output contract."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated the $12,824 of Social Security dependent benefits as the child's income, so it left out the $10,900.40 (85%) taxable to the head. It also capped the educator expense at $300 and deducted $275 of auto loan interest that the reference does not allow. It then submitted $3,305, which does not match its own computed $1,925."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It deducted $275 of auto loan interest, which the reference does not allow, cutting taxable income to $48,137.40 instead of $48,412.90. It also ended the head-of-household 10% bracket at $17,000 instead of the 2026 threshold of $17,700, so its pre-credit tax was $5,436.49 instead of $5,455.55."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It counted the full $12,824 of Social Security instead of the 85% taxable amount ($10,900.40). It wrongly treated $22,768 of employer and other health premiums as above-the-line deductions, and it used a $14,600 single standard deduction instead of the $24,150 head-of-household amount. It also submitted its pre-credit tax figure without subtracting the $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It treated the household as married filing jointly even though there is no spouse; the correct status is head of household with a $24,150 standard deduction. It also left the $10,900.40 of taxable Social Security out of income, so its $2,900 is too low compared with $5,455.55 of tax minus the $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It excluded the Social Security dependent benefits as the child's income, so it left out $10,900.40 of taxable Social Security from the head's AGI. It also used outdated parameters: a $23,000 standard deduction instead of $24,150, a $17,000 10% bracket instead of $17,700, and a $2,000 CTC instead of the 2026 OBBBA amount of $2,200."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It left the $10,900.40 of taxable Social Security out of the head's AGI by treating the benefits as the child's. It then submitted an unexplained pre-credit tax of $4,673 without subtracting the child tax credit, even though its own reasoning reached $2,673 after the credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It wrongly said the Social Security benefits are not taxable at this income level. Provisional income of about $68,075 is far above the $34,000 head-of-household base, so 85% ($10,900.40) is taxable. It also used a $24,500 standard deduction and a $2,000 CTC instead of $24,150 and $2,200, and replaced its computed $4,119 with an unexplained $4,519."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It capped the educator expense at $300, but the reference deducts the full $337.50. That raised AGI by $37.50 to $72,600.40 and added $4.50 of tax at 12%; the rest of its calculation matches the reference."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA expired in 2026 and applied old rules: personal exemptions, a 15% bracket, an $8,300 single standard deduction, and a $1,000 CTC. OBBBA made the TCJA structure permanent, so the correct figures are a $24,150 head-of-household standard deduction and a $2,200 CTC. It also used single filing status instead of head of household and left out the $10,900.40 of taxable Social Security."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It wrongly treated the Social Security dependent benefits as nontaxable, leaving $10,900.40 out of AGI. It also used outdated figures: a $23,350 standard deduction, a $16,550 bracket, and a $2,000 CTC. It then replaced its own computed $2,271 with an unexplained $3,892."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It deducted $275 of auto loan interest, which the reference does not allow because no qualifying new-vehicle purchase is established. This lowered taxable income to $48,137 instead of $48,412.90 and tax by about $33; the rest of its calculation matches the reference."
-us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used a $23,100 head-of-household standard deduction instead of the 2026 amount of $24,150 and a $2,000 CTC instead of the OBBBA amount of $2,200. It also capped the educator expense at $300 instead of allowing $337.50, and rounded the tax to $5,600."
-us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA rules with personal exemptions, an $11,000 standard deduction, a 15% bracket, and a $1,000 CTC, instead of the OBBBA figures of $24,150 and $2,200. It also subtracted $1,560 of non-employer health premiums from wages and left the $10,900.40 of taxable Social Security out of AGI."
-us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used a $23,100 head-of-household standard deduction instead of the 2026 amount of $24,150 and a $2,000 CTC instead of $2,200. It also capped the educator expense at $300 instead of $337.50, which left taxable income about $1,088 too high."
-us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules with a $12,200 standard deduction, $10,600 of personal exemptions, a 15% bracket, and a $1,000 CTC. OBBBA made the TCJA structure permanent, so the correct figures are a $24,150 head-of-household standard deduction, no exemptions, a 12% bracket, and a $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA expired and applied a $16,500 standard deduction, $10,100 of personal exemptions, a 15% bracket, and a $1,000 CTC instead of the OBBBA figures of $24,150 and $2,200. It also wrongly subtracted the $19,648 of employer-sponsored insurance premiums from the $62,000 of wages, but the reference keeps the full wages in gross income."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted the full $12,824 of Social Security instead of the 85% taxable amount ($10,900.40) and ignored the educator expense. It used a $15,300 standard deduction instead of the $24,150 head-of-household amount and a $2,000 CTC instead of $2,200, and its final number does not follow from those inputs."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It applied TCJA-expiration rules with personal exemptions, a 15% bracket, and a $1,000 CTC. OBBBA made the TCJA structure permanent for 2026, so the correct figures are a $24,150 standard deduction, no exemptions, the 10%/12% brackets, and a $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules with a $15,000 standard deduction, $10,100 of personal exemptions, and a $1,000 CTC instead of the OBBBA figures of $24,150 and $2,200. It also wrongly subtracted $19,648 of employer insurance premiums from wages."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct result is $5,455.55 of tax on $48,412.90 of taxable income minus the $2,200 CTC. Its $3,043 is $212.55 lower, which means it understated taxable income by about $1,770 or used wrong bracket parameters."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted a $2,000 CTC instead of the 2026 OBBBA amount of $2,200. Its implied pre-credit tax of $5,678 is $222.45 above the reference's pre-credit tax of $5,455.55."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It applied personal exemptions, which OBBBA permanently set to zero, so it took deductions beyond the $24,150 head-of-household standard deduction. As a result its answer is $1,366.81 below the correct $3,255.55."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the pre-OBBBA $22,500 head-of-household standard deduction instead of $24,150, the $17,000 10% bracket instead of $17,700, and a $2,000 CTC instead of $2,200. It also capped the educator expense at $300 instead of $337.50."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It correctly used the $2,200 CTC. Its implied pre-credit tax of $5,557.58 is $102.03 above the correct $5,455.55, which means it overstated taxable income by about $850 by using a lower standard deduction or bracket threshold than the 2026 head-of-household values of $24,150 and $17,700."
-us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It treated the Social Security dependent benefits as the child's untaxed income, leaving the head's $10,900.40 of taxable Social Security out of AGI. It also used a $22,500 standard deduction instead of $24,150 and a $2,000 CTC instead of $2,200."
-us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Its AGI of $72,562 was correct, but it computed $6,347 of tax on about $48,212 of taxable income. The 2026 head-of-household 10%/12% brackets give about $5,431 on that amount, so it overstated pre-credit tax by roughly $900. It also used a $24,350 standard deduction instead of $24,150."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and CTC wipe out all tax. Taxable income is $48,412.90 and produces $5,455.55 of tax, while the only nonrefundable credit is the $2,200 CTC, which leaves $3,255.55."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It subtracted medical expenses as deductions, but they are not deductible here because they fall below the 7.5%-of-AGI floor and the standard deduction is far larger than any itemized deductions. It also never added the 85% taxable Social Security ($10,900.40) or applied the full $2,200 CTC to $5,455.55 of tax."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It wrongly limited the nonrefundable CTC to $384.40 by setting aside a 'refundable portion' first. The nonrefundable CTC applies up to the tax liability, so the full $2,200 is used. It also left out the $10,900.40 of taxable Social Security and used a $22,500 standard deduction instead of $24,150."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It counted only 50% of the benefits ($6,412) as taxable Social Security. Provisional income of about $68,075 is above the $34,000 head-of-household upper base, so the 85% cap applies and $10,900.40 is taxable."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It used the 2025 head-of-household 10% bracket threshold of $17,000 instead of the 2026 threshold of $17,700. That gives $5,469.55 of tax instead of $5,455.55, so its answer is $13.45 too high after the $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It left the $10,900.40 of taxable Social Security dependent benefits out of AGI, which gave taxable income of $37,550 instead of $48,412.90. It also capped the educator expense at $300 instead of allowing $337.50."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It capped the educator expense at $300, but the reference deducts the full $337.50. That left taxable income $37.50 too high and tax $4.50 too high."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed a mortgage interest deduction even though only a mortgage balance was given, so the interest is $0, and it combined that with the standard deduction. It also left the $10,900.40 of taxable Social Security out of its stated income, and its $4,523 is $1,267.45 above the correct $5,455.55 minus $2,200."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied TCJA-expiration rules with a $12,389 standard deduction, $10,733 of personal exemptions, a 15% bracket, and a $1,000 CTC. OBBBA made the TCJA structure permanent, so the correct figures are a $24,150 head-of-household standard deduction, no exemptions, a 12% bracket, and a $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It applied sunset rules with a $12,400 standard deduction, two $5,350 exemptions, a 15% bracket, and a $1,000 CTC. Under OBBBA, 2026 uses a $24,150 head-of-household standard deduction, no exemptions, the 10%/12% brackets, and a $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It capped the educator expense at $300 instead of $337.50 and miscomputed 85% of $12,824 as $10,890 instead of $10,900.40. It also stated $5,464 of tax on $48,440, but the 2026 head-of-household brackets give $5,458.80 on that amount."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It applied post-TCJA-expiration rules with a $12,538 standard deduction, $10,862 of personal exemptions, a 15% bracket, and a $1,000 CTC. OBBBA made the TCJA structure permanent, so the correct figures are a $24,150 standard deduction and a $2,200 CTC."
-us,scenario_086,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It capped the educator expense at $300 instead of $337.50 and computed 85% of $12,824 as $10,890 instead of $10,900.40. It then rounded the pre-credit tax to $5,460 instead of the correct $5,455.55."
-us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value for federal_income_tax_before_refundable_credits and gave no explanation, so there is no answer to evaluate."
-us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted $275 of auto loan interest from AGI, but the reference allows no auto-loan-interest deduction because no qualifying new-vehicle purchase is established. That lowered taxable income to $48,137.40 instead of $48,412.90 and tax by $33.06."
-us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It used single filing status with a $15,000 standard deduction and single brackets instead of head of household with a $24,150 standard deduction and a $17,700 10% bracket. It also left out the $10,900.40 of taxable Social Security and used a $2,000 CTC instead of $2,200."
-us,scenario_086,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It left out the $337.50 educator expense deduction, which put taxable income at $48,750 instead of $48,412.90. It also ended the 10% bracket at $17,000 instead of the 2026 head-of-household threshold of $17,700."
-us,scenario_086,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Its $49,275 of taxable income equals $72,900 of gross income minus the 2025 head-of-household standard deduction of $23,625. It therefore left out the $337.50 educator expense and used 2025 values instead of the 2026 $24,150 standard deduction, and it ended the 10% bracket at $17,000 instead of $17,700."
-us,scenario_086,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It filed as single with a $16,100 standard deduction instead of head of household with $24,150, and it deducted $275 of auto loan interest. It also applied only $500 of the CTC as nonrefundable. In fact the nonrefundable CTC applies first up to the tax liability, so the full $2,200 offsets tax."
-us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It filed as married filing jointly even though there is no spouse, using a $31,200 standard deduction and joint brackets instead of head of household with $24,150 and a $17,700 10% bracket. It also used a $2,000 CTC instead of the OBBBA amount of $2,200."
-us,scenario_086,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used the $32,200 married-filing-jointly standard deduction instead of the $24,150 head-of-household amount. It also wrongly claimed that a Social Security exclusion reduces taxable income to $0, when 85% ($10,900.40) is taxable and taxable income is $48,412.90, which produces $3,255.55 of tax after the $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated the Social Security dependent benefits as the child's income, which left the $10,900.40 taxable portion out of the Head's AGI. It also capped the educator deduction at $300 and subtracted $275 of auto-loan interest that doesn't qualify. It then submitted $3,305, which contradicts its own computed $1,925."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its taxable Social Security and AGI were essentially correct. However, it subtracted a $275 OBBBA car-loan interest deduction that doesn't apply, cutting tax by $33. It also ended the 10% head-of-household bracket at $17,000 instead of the 2026 figure of $17,700, which netted a $19 shortfall."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"Its brackets, taxable Social Security and standard deduction were all correct. However, it subtracted $275 of auto-loan interest as an OBBBA deduction, but this household has no qualifying new-vehicle loan, so taxable income is $48,412.90, not $48,137.40. That understated tax by $33."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used single filing status with the 2024 $14,600 standard deduction instead of the $24,150 head-of-household deduction. It deducted $19,648 of employer premiums and $3,120 of other premiums above the line, and counted the full $12,824 of benefits instead of 85%. It then submitted pre-credit tax without subtracting the $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It computed tax as if the household filed jointly, but a single parent with a child files as head of household. It also left the $10,900.40 of taxable Social Security benefits out of income, which understated taxable income."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It excluded the Social Security dependent benefits as the child's income, but $10,900.40 (85%) is taxable on the Head's return. It also used an outdated $23,000 head-of-household standard deduction, a $17,000 10% bracket and a $2,000 CTC instead of $24,150, $17,700 and $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It excluded the $10,900.40 of taxable Social Security benefits as the child's income and used an outdated $23,000 standard deduction. It then submitted $4,673, a pre-credit figure it never derived, instead of the post-CTC amount its own reasoning reached."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It treated the Social Security dependent benefits as nontaxable, omitting the $10,900.40 that is includible at this income level. It used a $24,500 standard deduction and a $2,000 CTC instead of $24,150 and $2,200, and arbitrarily inflated its own $4,119 tax to $4,519."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"Everything except the educator deduction was correct. It capped that deduction at $300, but the inflation-indexed 2026 limit exceeds the $337.50 spent, so the full amount is deductible. That overstated taxable income by $37.50 and tax by $4.50."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA would sunset in 2026 and applied pre-TCJA single-filer brackets, an $8,300 standard deduction, personal exemptions and a $1,000 CTC. OBBBA made the TCJA structure permanent, with a $24,150 head-of-household deduction and a $2,200 CTC. It also excluded the $10,900.40 of taxable Social Security benefits as the child's income."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It treated the Social Security benefits as nontaxable, but 85% ($10,900.40) is taxable because provisional income far exceeds $34,000. It used a stale $23,350 standard deduction and a $2,000 CTC, then submitted $3,892, which contradicts its own computed $2,271."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It deducted $275 of auto-loan interest even though the household has no qualifying new-vehicle loan, cutting taxable income to $48,137 instead of $48,412.90. That understated tax by $33."
+us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used a $23,100 head-of-household standard deduction instead of the OBBBA 2026 figure of $24,150, and a $2,000 CTC instead of $2,200. It also capped the educator deduction at $300 and rounded the bracket tax to $5,600 instead of computing $5,455.55."
+us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA law, with an $11,000 standard deduction, $9,600 of personal exemptions, a 15% bracket and a $1,000 CTC. Under OBBBA, 2026 uses a $24,150 head-of-household deduction, no exemptions, 10%/12% brackets and a $2,200 CTC. It also omitted the $10,900.40 of taxable Social Security and wrongly subtracted $1,560 of premiums from wages."
+us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used a $23,100 head-of-household standard deduction instead of the OBBBA 2026 figure of $24,150, and a $2,000 CTC instead of $2,200. It also capped the educator deduction at $300. Together these overstated tax by $335.50."
+us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA sunset rules, with a $12,200 standard deduction, two personal exemptions, a 15% bracket and a $1,000 CTC. For 2026, OBBBA retains a $24,150 head-of-household deduction, no exemptions, 10%/12% brackets and a $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA would expire, so it used personal exemptions, a 15% bracket and a $1,000 CTC instead of OBBBA's $24,150 deduction and $2,200 CTC. It also subtracted the $19,648 of employer-sponsored premiums from the $62,000 of wages, even though the wage figure is already the taxable amount."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It counted the full $12,824 of benefits as income instead of the 85% taxable portion ($10,900.40), and used a $15,300 standard deduction instead of the $24,150 head-of-household amount. It also used a $2,000 CTC instead of $2,200, and its $1,782 doesn't follow from its own inputs."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA individual provisions would expire in 2026, so it applied personal exemptions, the 15% bracket and a $1,000 nonrefundable CTC. OBBBA made the $24,150 head-of-household standard deduction, the 10%/12% brackets and the $2,200 CTC permanent."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It applied post-TCJA-expiration rules, with a $15,000 deduction, $10,100 of personal exemptions and a $1,000 CTC, instead of OBBBA's $24,150 head-of-household deduction and $2,200 CTC. It also subtracted the $19,648 of employer premiums from wages, which shouldn't be done."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $3,043 is $212.55 below the correct $3,255.55, which equals about $1,771 of extra deductions at the 12% rate. Its taxable income therefore falls short of the $48,412.90 from AGI of $72,562.90 minus the $24,150 head-of-household deduction."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used the pre-OBBBA $22,500 head-of-household standard deduction instead of $24,150, and a $2,000 CTC instead of $2,200. Its $3,678 implies pre-credit tax of about $5,678 on taxable income near $50,100."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It subtracted personal exemptions on top of the head-of-household standard deduction, but exemptions are $0 under OBBBA. That pulled tax down to $1,888.74. The correct figure is $5,455.55 of tax on $48,412.90 of taxable income, less the $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used the pre-OBBBA $22,500 head-of-household standard deduction instead of $24,150, a $17,000 10% bracket instead of $17,700, and a $2,000 CTC instead of $2,200. It also capped the educator deduction at $300."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It applied the correct $2,200 CTC, but its implied pre-credit tax of $5,557.58 is $102.03 too high. That corresponds to about $850 too much taxable income, so it understated the deductions that bring AGI of $72,562.90 down to $48,412.90."
+us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It treated the Social Security dependent benefits as the child's nontaxable income, omitting the $10,900.40 taxable on the Head's return. It also used a $22,500 standard deduction and a $2,000 CTC instead of OBBBA's $24,150 and $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"Its AGI of $72,562 was correct. It then used a $24,350 deduction instead of $24,150 and miscomputed the bracket tax as $6,347 on $48,212, when the 10%/12% head-of-household brackets yield about $5,431. That overstated tax by roughly $900."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction and CTC wipe out all tax. In fact, $48,412.90 of taxable income generates $5,455.55 of tax, and the $2,200 nonrefundable CTC leaves $3,255.55."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It built taxable income from wages alone, omitting the $10,900.40 of taxable Social Security. It also subtracted itemized medical expenses that fall far below the 7.5%-of-AGI floor, while the $24,150 head-of-household standard deduction applies anyway."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It omitted the $10,900.40 of taxable Social Security and used the stale $22,500 standard deduction. It then subtracted only $384.40 of CTC by carving out a 'refundable portion' first. With $4,364 of tax, the full $2,200 CTC is absorbed as a nonrefundable credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It counted only $6,412 (50%) of the benefits as taxable. With provisional income of about $68,075, far above the $34,000 head-of-household second threshold, the 85% cap applies, making $10,900.40 taxable."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $3,269 is $13.45 above the correct figure, which is about $112 too much taxable income. That matches capping the educator deduction at $300 instead of allowing the full $337.50, plus rounded inputs rather than the exact $48,412.90 of taxable income."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"Its deduction, brackets and CTC were correct, but its taxable income of $37,550 leaves out the $10,900.40 of taxable Social Security benefits. It also capped the educator deduction at $300."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"Its result of $3,260.05 matches capping the educator deduction at $300. The inflation-indexed 2026 limit exceeds the $337.50 spent, so the full amount is deductible; the cap overstated tax by $4.50."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It invoked a mortgage interest deduction, but only a mortgage balance is listed, so interest is $0. The household takes the $24,150 head-of-household standard deduction. Its $4,523 overstates tax by $1,267 relative to $5,455.55 of tax on $48,412.90 less the $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied TCJA-expiration rules, with a $12,389 deduction, $10,733 of personal exemptions, a 15% bracket and a $1,000 CTC. Under OBBBA, 2026 uses the $24,150 head-of-household deduction, no exemptions, 10%/12% brackets and a $2,200 CTC."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It assumed a 2026 TCJA sunset, applying a $12,400 deduction, two $5,350 exemptions, a 15% bracket and a $1,000 CTC. OBBBA keeps the $24,150 head-of-household deduction, the 10%/12% brackets and a $2,200 CTC, with no exemptions."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It capped the educator deduction at $300 instead of allowing the full $337.50, and rounded taxable Social Security to $10,890. It also overstated the bracket tax on $48,440 as $5,464 when it is $5,458.80, which produced $3,264."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It applied post-TCJA-expiration rules, with a $12,538 deduction, two personal exemptions, a 15% bracket and a $1,000 CTC. OBBBA made the $24,150 head-of-household deduction, the 10%/12% brackets and the $2,200 CTC permanent for 2026."
+us,scenario_086,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It capped the educator deduction at $300 instead of deducting the full $337.50, and rounded taxable Social Security and the tax. That gave $3,260 instead of $3,255.55."
+us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value for federal_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted $275 of auto-loan interest even though the household has no qualifying new-vehicle loan. That reduced taxable income to $48,137.40 instead of $48,412.90 and understated tax by $33."
+us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It used single filing status and a $15,000 standard deduction instead of head of household and $24,150. It also omitted the $10,900.40 of taxable Social Security and used a $2,000 CTC instead of $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It never subtracted the $337.50 educator expense, so its AGI was about $72,900 instead of $72,562.90. It then overstated the bracket tax as $5,509 on $48,750 of taxable income."
+us,scenario_086,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Its taxable income of $49,275 matches the 2025 $23,625 head-of-household deduction with no educator deduction. Its $5,573 tax uses the 2025 $17,000 10% bracket. For 2026, the deduction is $24,150, the bracket is $17,700, and $337.50 of educator expense is deductible."
+us,scenario_086,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It filed the Head as single with a $16,100 deduction and pushed income into the 22% bracket, instead of head of household with $24,150 and a 12% top rate. It also deducted $275 of car-loan interest and allowed only $500 of the CTC as nonrefundable, though the full $2,200 is usable against the tax."
+us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It treated a parent and child as married filing jointly, using MFJ Social Security thresholds, a $31,200 deduction and MFJ brackets. The correct status is head of household, with a $24,150 deduction and the 10% bracket ending at $17,700. It also used a $2,000 CTC instead of $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It applied a $32,200 joint-filer deduction and excluded the Social Security benefits, then claimed taxable income of $0. Head-of-household filing with $10,900.40 of taxable benefits gives $48,412.90 of taxable income and $3,255.55 of tax after the CTC."
us,scenario_086,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model estimated tax of about $4,300 and a CTC of $2,200, then set ACTC to min($1,700, 15% × (earnings − $2,500)). It left out the limit that ACTC cannot exceed the CTC remaining after the nonrefundable offset. Because $4,300 of tax absorbs the whole $2,200, the refundable portion is $0."
us,scenario_086,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model said the tax liability is 'largely offset' by the CTC and still assigned the full $1,700 ACTC cap. The refundable amount is limited to CTC left unused after tax, and pre-credit tax of more than $4,000 uses up the entire credit. It also used the old $2,000 per-child amount instead of $2,200."
us,scenario_086,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model correctly found that tax of about $4,300 absorbs the full CTC and leaves no ACTC. It then reversed itself by inventing extra deductions that do not exist in the facts and submitted $1,700. The correct conclusion from its own reasoning is $0."
@@ -7805,56 +7888,57 @@ us,scenario_086,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"T
us,scenario_086,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct component amounts—$3,844 of Social Security tax and $899 of Medicare tax—but failed to add them correctly. Those components total $4,743, not $3,979, and no Georgia-specific adjustment changes that sum."
us,scenario_086,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract."
us,scenario_086,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated the correct $3,844 Social Security tax and $899 Medicare tax but submitted $3,813 instead of adding them. The stated components total $4,743."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It applied a $24,000 standard deduction (the 2024–2025 joint-filer amount) instead of the $15,000 head-of-household amount, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%. That gave $1,749. It then submitted $2,732, a number its own working never produces."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It subtracted the $275 of auto loan interest from GA AGI, but personal interest is not deductible. It also used a $24,000 standard deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.09% rate instead of 4.99%, which overstated deductions and gave $1,699.40."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied Georgia's repealed graduated 1%–6% bracket schedule to the full $62,000 of wages. It took no standard deduction or dependent exemption, instead of applying the 4.99% flat rate to $41,662.50 of taxable income."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It used a 5.19% rate instead of 4.99%. Its $2,900 answer implies about $55,900 of taxable income, so it subtracted only about $6,000 rather than the $15,000 standard deduction plus the $5,000 dependent exemption."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used an $18,500 head-of-household standard deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%. That gave $2,032.51, and it then raised the answer to $2,191 with no stated basis."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It treated the head as a single filer with a $12,000 standard deduction instead of a head-of-household filer with $15,000. It also used a $4,000 dependent exemption instead of $5,000 and a 5.09% rate instead of 4.99%, which gave $2,324, and then submitted an unexplained $1,869."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It subtracted a $12,000 standard deduction instead of $15,000 and a $4,000 dependent exemption instead of $5,000, giving $45,700 of taxable income instead of $41,662.50. It then taxed that at an assumed 5.09% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Its federal AGI already excluded Social Security, yet it subtracted the full $12,824 again. It then used the repealed pre-2024 structure ($5,400 standard deduction, $2,700 personal exemption, $3,000 dependent exemption) and a 5.39% rate instead of the $15,000 deduction, $5,000 dependent exemption and 4.99% rate. These errors offset to $2,034."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a single-filer $12,000 standard deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%. That gave $2,372, and it then inflated the answer to $2,765 with no basis."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It subtracted a $12,000 standard deduction instead of $15,000 and a $4,000 dependent exemption instead of $5,000, leaving $45,662 instead of $41,662.50. It then applied 5.09% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It started from $72,600 of federal AGI without Georgia's subtraction of the $10,900.40 of taxable Social Security. It then used a $9,000 standard deduction and two $2,700 personal exemptions instead of $15,000 plus a $5,000 dependent exemption, and taxed the result at 5.29% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used the pre-2024 $5,400 standard deduction and a $2,700 exemption for each of the head and child instead of the $15,000 standard deduction and $5,000 dependent exemption. It started from a $60,140 AGI instead of the $61,662.50 GA AGI and taxed the result at 5.29% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a $5,700 standard deduction, a $2,700 filer exemption and a $3,000 child exemption, leaving $50,300 taxable. The correct deductions are a $15,000 standard deduction and a $5,000 dependent exemption, leaving $41,662.50, and it used 5.09% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used the correct 4.99% rate. However, it started from the full $62,000 of wages and skipped the educator expense deduction, used a $12,000 standard deduction instead of $15,000, and claimed $6,700 of personal and dependent exemptions instead of the single $5,000 dependent exemption, ending at $43,300 instead of $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It started from a $52,952.40 AGI, well below the $61,662.50 GA AGI (wages minus educator expenses, with Social Security excluded). It then used a $15,800 head-of-household amount and a $3,000 dependent exemption instead of $15,000 and $5,000, and taxed the result at 5.39% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $2,690 matches roughly $50,000 of taxable income ($62,000 of wages minus a $12,000 deduction) at about 5.39%. It omitted the $5,000 dependent exemption, under-applied the $15,000 standard deduction, and used a rate above the 4.99% flat rate."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It correctly derived $61,662 of GA AGI but subtracted a $12,000 standard deduction instead of $15,000 and a $3,000 dependent exemption instead of $5,000. It then applied 5.39% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It started from a $52,952 AGI instead of the $61,662.50 GA AGI, understating income by about $8,700. It then applied a 5.29% rate instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $3,212 corresponds to roughly the entire $61,662 GA AGI taxed at about 5.2%. That omits the $15,000 standard deduction and the $5,000 dependent exemption, and uses a rate above 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $3,047 implies roughly $57,000–$61,000 of taxable income at a 5%-range rate. It therefore subtracted far less than the $20,000 of combined standard deduction and dependent exemption from the $61,662.50 GA AGI, and did not apply the 4.99% rate to $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $1,432.32 implies only about $27,000–$29,000 of taxable income. It therefore over-deducted from the $61,662.50 GA AGI, well beyond the $15,000 standard deduction plus $5,000 dependent exemption that leave $41,662.50 at 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It subtracted a $12,000 standard deduction instead of $15,000 and a $3,000 dependent exemption instead of $5,000, leaving $46,700 instead of $41,662.50. It then applied 5.39% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"Its $2,236.29 implies about $43,000–$45,000 of taxable income at a rate around 5%. That reflects deductions below the $15,000 standard deduction plus $5,000 dependent exemption and a rate other than the 4.99% applied to $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $10,600 head-of-household standard deduction instead of $15,000 and omitted the $5,000 dependent exemption entirely, leaving $51,062 taxable instead of $41,662.50. It then applied 5.39% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $24,000 standard deduction (the 2024–2025 joint-filer amount) instead of $15,000 and about $8,400 of exemptions instead of the single $5,000 dependent exemption. That understated taxable income at $29,262, and it then applied 5.19% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $1,639 implies only about $32,000 of taxable income at a roughly 5% rate. It therefore over-deducted from the $61,662.50 GA AGI instead of subtracting exactly the $15,000 standard deduction and the $5,000 dependent exemption, then taxing $41,662.50 at 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It built Georgia income from federal-style taxable income with extra medical and other adjustments. Its $498 implies only about $10,000 of taxable income, instead of $41,662.50 (GA AGI minus the $15,000 standard deduction and $5,000 dependent exemption) taxed at 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It subtracted a $12,000 standard deduction instead of $15,000 and $4,500 of exemptions instead of the $5,000 dependent exemption, leaving $45,162. It then applied 5.19% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its $1,357.80 implies only about $27,000 of taxable income. It therefore applied far more than the $15,000 head-of-household standard deduction plus $5,000 dependent exemption to the $61,662.50 GA AGI."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It subtracted a $12,000 deduction instead of the $15,000 head-of-household standard deduction and a $4,000 dependent exemption instead of $5,000, leaving $45,662.40. It then applied 5.09% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It applied a 5.19% rate instead of 4.99%. Its $1,903 implies about $36,700 of taxable income, meaning it over-deducted by about $5,000 compared with the $41,662.50 left after the $15,000 standard deduction and $5,000 dependent exemption."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It subtracted a $12,000 standard deduction instead of $15,000 and a $4,000 dependent exemption instead of $5,000, leaving $45,662. It then applied 5.09% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used the correct 4.99% rate. However, it applied a $12,000 standard deduction instead of $15,000, invented a $4,000 taxpayer exemption that Georgia does not allow, and used a $3,000 dependent exemption instead of $5,000. That left $42,700 taxable instead of $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It subtracted a $12,000 deduction instead of the $15,000 head-of-household standard deduction and a $4,000 dependent exemption instead of $5,000, leaving $45,662. It then applied 5.19% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"It correctly started from $61,662 of GA AGI but used a $12,000 head-of-household deduction instead of $15,000 and a $4,000 dependent exemption instead of $5,000. It then applied 5.09% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It applied 5.19% instead of 4.99%. Its $2,371.83 corresponds to about $45,700 of taxable income, reflecting a $12,000 standard deduction and $4,000 dependent exemption instead of $15,000 and $5,000."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"Its $2,184 implies about $42,000–$44,000 of taxable income at a rate in the 5% range. That reflects deductions below the $15,000 standard deduction plus $5,000 dependent exemption and a rate above 4.99%, instead of $41,662.50 at 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It subtracted an $18,000 head-of-household standard deduction instead of $15,000 and omitted the $5,000 dependent exemption, leaving $43,662. It then applied 5.29% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used a $12,000 standard deduction instead of $15,000 and $5,700 of personal and dependent exemptions instead of the single $5,000 dependent exemption, leaving $44,000. It then applied 5.39% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It subtracted a $12,000 head-of-household standard deduction instead of $15,000 and a $3,000 dependent exemption instead of $5,000, leaving $46,700. It then applied 5.19% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It subtracted the $338 educator expense a second time even though federal AGI already reflected it. It used the repealed $5,400 standard deduction and two $2,700 exemptions instead of $15,000 plus a $5,000 dependent exemption, and applied the old 5.49% rate instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It reached about $40,700 of taxable income, which is about $1,000 below the correct $41,662.50 after the $15,000 standard deduction and $5,000 dependent exemption. It then applied a 5.29% rate instead of Georgia's 2026 rate of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It used the correct 4.99% rate and $61,662 GA AGI. However, it subtracted a $12,000 standard deduction instead of the $15,000 head-of-household amount and two $2,700 personal exemptions instead of the single $5,000 dependent exemption, leaving $44,262 instead of $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It understated GA AGI at $61,387 instead of $61,662.50 and used an $18,500 head-of-household deduction and a $3,000 dependent exemption instead of $15,000 and $5,000. It then applied 5.19% instead of 4.99%, and these errors partly offset to $2,070.14."
-us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It used a $5,000 standard deduction instead of $15,000 and a $2,700 personal exemption, and it omitted the $5,000 dependent exemption, leaving $54,000 taxable instead of $41,662.50. It then applied 5.19% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It started from about $72,900 of AGI, which still includes the $10,900.40 of taxable Social Security that Georgia subtracts. It then used about $18,000 of deductions instead of $20,000 and a 5.19% rate instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It taxed about $41,000 of taxable income, close to the correct $41,662.50, at a 5.19% rate. Georgia's 2026 rate is 4.99%, which overstated the tax."
-us,scenario_086,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used a single-filer $12,000 standard deduction instead of the $15,000 head-of-household amount, invented a $3,700 personal exemption, and used a $4,000 dependent exemption instead of $5,000. It then applied 5.09% instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It included the full $12,824 of Social Security benefits in Georgia income, although Georgia excludes Social Security entirely. It also used the old $2,700 and $3,000 exemptions with a $12,000 deduction instead of $15,000 plus $5,000, and a 5.12% rate instead of 4.99%."
-us,scenario_086,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It claimed to exclude Social Security but started from $74,824 of gross income, which includes the benefits. It then subtracted a $23,100 deduction instead of the $15,000 standard deduction plus $5,000 dependent exemption from the $61,662.50 GA AGI, and applied an effective 4.57% rate that matches no Georgia schedule."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used a $24,000 HoH standard deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%, arriving at $1,749. It then submitted $2,732 under a 'single-filer' label that contradicts its own computation. The correct figure is $61,662.50 − $15,000 − $5,000 = $41,662.50 × 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It subtracted the $275 auto loan interest from Georgia AGI, although that interest is not a Georgia subtraction. It also used a $24,000 HoH standard deduction (correct: $15,000), a $4,000 dependent exemption (correct: $5,000) and a 5.09% rate (correct: 4.99%). The result was $33,387 of taxable income instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,thresholds_rates,False,"Its Georgia AGI of $61,662 was correct. It then applied a $12,000 standard deduction instead of the $15,000 HoH amount and a $4,000 dependent exemption instead of $5,000, so taxable income was $45,662 instead of $41,662.50. It also taxed at 5.09% instead of the 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It applied a repealed graduated bracket schedule topping out at 6% to the full $62,000 of wages. It took no standard deduction or dependent exemption and did not apply the educator adjustment. Georgia's 2026 tax is a 4.99% flat rate on $41,662.50, after the $15,000 HoH standard deduction and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It used a 5.19% rate instead of 4.99%. Its $2,900 result implies about $55,900 of taxable income, so it did not fully apply the $15,000 HoH standard deduction and $5,000 dependent exemption to the $61,662.50 Georgia AGI."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used an $18,500 HoH standard deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%, getting $2,032.51. It then submitted an unexplained $2,191 that does not match that computation."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It treated the head as a single filer with a $12,000 standard deduction (correct: the $15,000 HoH amount), took a $4,000 dependent exemption instead of $5,000, and applied 5.09% instead of 4.99%. That gave $2,324, which it then changed to an arbitrary $1,869."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It applied a $12,000 standard deduction instead of the $15,000 HoH deduction and a $4,000 dependent exemption instead of $5,000. It also used a 5.09% rate instead of the 2026 rate of 4.99%, which left $45,700 taxable instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Its federal AGI left out the $10,900.40 of taxable Social Security, yet it still subtracted the full $12,824 in benefits, so it removed Social Security that was never in its AGI. It also used the repealed $5,400 single standard deduction, a $2,700 personal exemption and a $3,000 dependent exemption, plus a 5.39% rate. The correct parameters are the $15,000 HoH deduction, a $5,000 dependent exemption and 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $12,000 single standard deduction instead of the $15,000 HoH amount, a $4,000 dependent exemption instead of $5,000, and a 5.19% rate instead of 4.99%, computing about $2,372. It then inflated that to an unsupported $2,765."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"Its $61,662 Georgia income was correct. It then subtracted a $12,000 standard deduction instead of the $15,000 HoH deduction and a $4,000 dependent exemption instead of $5,000, and applied 5.09% instead of the 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It started from $72,600 of federal AGI without subtracting the $10,900.40 of taxable Social Security, which Georgia fully exempts. It also used an obsolete $9,000 HoH deduction, two $2,700 personal exemptions and a 5.29% rate, rather than the $15,000 HoH deduction, $5,000 dependent exemption and 4.99% rate."
+us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied the repealed $5,400 standard deduction and two $2,700 personal exemptions to a misstated $60,140 AGI, then taxed at 5.29%. Georgia's 2026 computation is $61,662.50 − $15,000 HoH standard deduction − $5,000 dependent exemption, taxed at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a pre-reform $5,700 standard deduction plus a $2,700 filer exemption and a $3,000 child exemption, which left $50,300 taxable. The correct deductions are the $15,000 HoH standard deduction and a $5,000 dependent exemption, which leave $41,662.50. It also used 5.09% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used the correct 4.99% rate but started from gross wages of $62,000 without the educator adjustment. It then used a $12,000 standard deduction plus $6,700 of exemptions, including a personal exemption that no longer exists, instead of the $15,000 HoH deduction and $5,000 dependent exemption. Taxable income came out at $43,300 instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $19,648 of employer-sponsored insurance premiums from wages, although the employer pays those and they are not a reduction of gross wages. It also kept taxable Social Security in Georgia income instead of exempting it. On top of that it used a fictitious $15,800 HoH exemption, a $3,000 dependent exemption and the 5.39% rate, rather than $15,000 + $5,000 at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no derivation. $2,690 implies about $51,800 of taxable income at 5.19%, which means it took only about $10,000 off wages and used a rate above 4.99%. The correct figure is ($61,662.50 − $15,000 HoH standard deduction − $5,000 dependent exemption) × 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Its $61,662 Georgia AGI was correct. It then used a $12,000 HoH standard deduction instead of $15,000, a $3,000 dependent exemption instead of $5,000, and a 5.39% rate instead of the 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Its $52,952 'federal AGI' comes from subtracting the $19,648 employer insurance premiums from wages and leaving taxable Social Security in Georgia income, instead of using Georgia AGI of $61,662.50. It also applied 5.29% rather than 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation. $3,212 is about 5.19% of roughly $61,900, which means it taxed essentially all of Georgia AGI without the $15,000 HoH standard deduction and $5,000 dependent exemption, and used a rate above the 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. $3,047 matches federal AGI of $72,562.90 minus $12,000 and $4,000, taxed at about 5.39%. That means it left the $10,900.40 of taxable Social Security in Georgia income and used outdated deduction, exemption and rate parameters instead of $15,000, $5,000 and 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave no derivation. $1,432.32 implies only about $27,600–$28,700 of taxable income, so it deducted roughly $33,000 from the $61,662.50 Georgia AGI. Georgia allows only the $15,000 HoH standard deduction and the $5,000 dependent exemption, which leave $41,662.50 taxed at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its Georgia AGI of about $61,700 was right. It then subtracted a $12,000 HoH standard deduction instead of $15,000 and a $3,000 dependent exemption instead of $5,000, and taxed at 5.39% instead of the 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"It correctly excluded Social Security. However, $2,236.29 implies about $43,000–$44,800 of taxable income, so its standard deduction and dependent exemption added up to less than the required $15,000 + $5,000, and/or it used a rate above 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Its $61,662 Georgia AGI was correct. It then used a $10,600 HoH standard deduction instead of $15,000, left out the $5,000 dependent exemption for the child entirely, and applied 5.39% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It reached Georgia AGI correctly but then over-deducted. It used a $24,000 HoH standard deduction (correct: $15,000) plus about $8,400 of personal and dependent exemptions (correct: one $5,000 dependent exemption). It also used 5.19% instead of 4.99%, leaving $29,262 taxable instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It gave no derivation. $1,639 implies about $31,600–$32,800 of taxable income, roughly $9,000–$10,000 below the $41,662.50 left after the $15,000 HoH standard deduction and $5,000 dependent exemption. That shows over-stated Georgia deductions and exemptions."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It built Georgia taxable income from a federal-style taxable income with medical adjustments, which Georgia does not use. $498 implies only about $10,000 of taxable income, instead of $61,662.50 − $15,000 − $5,000 = $41,662.50 taxed at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It used a $12,000 HoH standard deduction instead of $15,000 and $4,500 of personal and dependent exemptions instead of the single $5,000 dependent exemption, which left $45,162 taxable. It then applied 5.19% instead of the 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly excluded taxable Social Security. However, $1,357.80 implies only about $26,000–$27,000 of taxable income, meaning it deducted about $35,000 instead of the $15,000 HoH standard deduction plus the $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It applied $12,000 and $4,000 in place of the $15,000 HoH standard deduction and the $5,000 dependent exemption, which left $45,662.40 taxable instead of $41,662.50. It also taxed at 5.09% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It applied 5.19% instead of 4.99%. Its $1,903 result implies about $36,700 taxable, so it deducted roughly $25,000 instead of the $15,000 HoH standard deduction plus the $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It used a $12,000 standard deduction and a $4,000 dependent exemption instead of the 2026 amounts of $15,000 (HoH) and $5,000. It also used the 5.09% rate instead of 4.99%, which gave tax on $45,662 instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It had the correct 4.99% rate. It deducted a $12,000 standard deduction, a nonexistent $4,000 taxpayer exemption and a $3,000 dependent exemption, instead of the $15,000 HoH standard deduction and $5,000 dependent exemption. Taxable income came out at $42,700 instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It used a $12,000 HoH deduction instead of $15,000 and a $4,000 dependent exemption instead of $5,000, which left $45,662 taxable. It then applied 5.19% instead of the 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"Its $61,662 Georgia AGI was correct. It used a $12,000 HoH deduction instead of $15,000, a $4,000 dependent exemption instead of $5,000, and the 5.09% rate instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"$2,371.83 equals 5.19% × $45,700, which means it used $12,000 + $4,000 of deductions instead of the $15,000 HoH standard deduction and $5,000 dependent exemption. It also used 5.19% instead of the 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave no derivation. $2,184 implies about $42,000–$43,800 of taxable income at a 4.99–5.19% rate. That means it deducted less than the required $15,000 HoH standard deduction plus $5,000 dependent exemption from the $61,662.50 Georgia AGI, or used a rate above 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used an $18,000 HoH standard deduction instead of $15,000 and omitted the $5,000 dependent exemption entirely, which left $43,662 taxable. It also applied 5.29% instead of the 2026 rate of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It subtracted a $12,000 HoH deduction instead of $15,000 and $5,700 of personal and dependent exemptions, including a filer exemption that no longer exists, instead of the single $5,000 dependent exemption. It then taxed at 5.39% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Its $61,700 Georgia AGI was essentially right. It then used a $12,000 HoH standard deduction instead of $15,000 and a $3,000 dependent exemption instead of $5,000, and applied 5.19% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It subtracted the $338 educator expense a second time, although federal AGI already reflects it. It then applied the repealed $5,400 standard deduction, two $2,700 exemptions and the 2024 rate of 5.49%, rather than the $15,000 HoH deduction, $5,000 dependent exemption and 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its taxable income of about $40,700 was close to the correct $41,662.50, but it applied a 5.29% flat rate instead of Georgia's 2026 rate of 4.99%. That overstated the tax."
+us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"Its $61,662 Georgia AGI and 4.99% rate were correct. It subtracted a $12,000 standard deduction and two $2,700 personal exemptions instead of the $15,000 HoH standard deduction and $5,000 dependent exemption, leaving $44,262 taxable instead of $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It reduced Georgia AGI to $61,387 by removing auto loan interest, which is not a Georgia subtraction. It also used an $18,500 HoH standard deduction instead of $15,000, a $3,000 dependent exemption instead of $5,000, and 5.19% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It applied a $5,000 standard deduction and a $2,700 personal exemption and left out the child's dependent exemption. It taxed $54,000 at 5.19% instead of $41,662.50 at 4.99%, the result of the $15,000 HoH standard deduction and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It started from about $72,900 of AGI without subtracting the $10,900.40 of taxable Social Security, which Georgia fully exempts. It then used $12,000 + $6,000 of deductions instead of $15,000 + $5,000, and 5.19% instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"Its taxable income of about $41,000 was close to the correct $41,662.50, but it applied a 5.19% rate instead of Georgia's 2026 rate of 4.99%. That overstated the tax."
+us,scenario_086,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"Its taxable income of $41,687 nearly matched $41,662.50 only because errors offset: a $12,000 single deduction, a nonexistent $3,700 personal exemption and a $4,000 dependent exemption, applied to an AGI already reduced by auto loan interest. The deciding error was the 5.09% rate instead of 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It included the full $12,824 of Social Security benefits in Georgia income, although Georgia fully exempts them. It then used a $12,000 standard deduction, $2,700 and $3,000 exemptions and a 5.12% rate, rather than $61,662.50 − $15,000 − $5,000 taxed at 4.99%."
+us,scenario_086,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from gross income of $74,824, including Social Security, and subtracted a fictitious $23,100 standard deduction. It then applied an implied rate of about 4.57%. Georgia AGI is $61,662.50 after exempting taxable Social Security, minus the $15,000 HoH deduction and $5,000 dependent exemption, taxed at 4.99%."
us,scenario_086,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented a Georgia refundable dependent-child credit of $260 per qualifying child. Georgia has no such applicable refundable credit, so the 12-year-old dependent generates $0 in state refundable credits."
us,scenario_088,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the deduction for the $9,600 of qualified tips included in wages, leaving $7,085 of purported taxable income. It also incorrectly included the $675 state tax refund in gross income without establishing a prior-year itemized-deduction tax benefit."
us,scenario_088,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied neither a numeric value nor an explanation for the requested output, violating the required output contract."
@@ -8064,53 +8148,54 @@ us,scenario_090,state_refundable_credits,gpt-5.6-luna,llm_error,age_disability,F
us,scenario_090,state_refundable_credits,grok-4.5,llm_error,age_disability,False,"It computed the Kansas refundable EITC as 17% of a $536 federal EITC without applying the age-25 minimum for the childless federal EITC; at age 22 the head's federal EITC is $0, making the Kansas EITC $0. Its Kansas rate and arithmetic were right, but the federal base it multiplied was disallowed by the age test."
us,scenario_090,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"It applied only the Kansas food sales tax credit's income ceiling and skipped the categorical qualification, which limits the credit to filers who are 55 or older, blind or disabled, or have a dependent child under 18 living with them; a 22-year-old single filer with no dependents satisfies none of those, so the credit is $0. It further treated the credit as refundable, but Kansas made it nonrefundable beginning in tax year 2013, so even a qualifying $125 credit would not appear in state_refundable_credits."
us,scenario_090,tanf,glm-5.3,llm_error,categorical_eligibility,False,"Kansas TANF requires a dependent child in the assistance unit, and this 22-year-old head is a childless adult, so the benefit is $0 regardless of income; the household also holds $20,300 in bank assets, far above Kansas's TANF resource limit. glm-5.3 ran no eligibility test at all - it copied the $4,800 'financial assistance' input, a separate unearned-income variable in PolicyEngine rather than a TANF payment, straight into the tanf output. Its own stated reasoning ('financial assistance during the year, which is its annual TANF cash assistance amount') shows the substitution, and the prompt's instruction not to infer unlisted benefit receipt forbids reclassifying that input as TANF."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It left out the $628 qualified-overtime deduction, so its ordinary taxable income was $14,945 instead of $14,316.66 and its own working reached $1,425. It then submitted $1,276 without explaining the change from that $1,425."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It never applied the $628 qualified-overtime deduction. It also taxed the $1,170 of capital-gain distributions at ordinary rates, which pushed ordinary taxable income to $16,115 instead of $14,316.66. The result was $1,685.80 before credits instead of $1,470."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It used the 2024 standard deduction of $14,600, did not subtract the 401(k), taxed qualified dividends at 15% instead of 0%, and claimed a $2,500 AOTC even though no qualified tuition expenses are listed. It then submitted $650, a figure that does not follow from its own working."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It assumed the nonrefundable AOTC wipes out the tax, but no qualified tuition expenses are listed, so the AOTC is $0. The $1,470 of tax is reduced only by the $120 Saver's Credit, leaving $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It skipped the $628 qualified-overtime deduction and taxed the $1,170 of gains at ordinary rates, which gave $1,686 of tentative tax. It also said no nonrefundable credits apply, missing the $120 Saver's Credit, and then submitted an unexplained $1,656."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"It did not subtract the $926 traditional 401(k) from the $27,000 of gross wages, skipped the $628 overtime deduction, and invented a $331 nonrefundable AOTC with no tuition expenses on record. It also never applied the $120 Saver's Credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Its own working reached about $1,684 without the $628 overtime deduction, and it then raised the answer to $2,109 for no stated reason. It also missed the $120 Saver's Credit; with the overtime deduction, ordinary tax is $1,470, and the credit brings it to $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction and a $11,925 top of the 10% bracket instead of the 2026 figures of $16,100 and $12,400. It also omitted the $628 qualified-overtime deduction and the $120 Saver's Credit, ending at $1,687 instead of $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"Its working used a $15,750 standard deduction and subtracted a $1,500 AOTC even though no tuition is listed. It then adjusted to $1,470, which equals the pre-credit tax but leaves out the $120 Saver's Credit that brings the liability to $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It handled the standard deduction, the 0% rate on dividends and gains, and the $120 Saver's Credit correctly, but it missed the $628 qualified-overtime deduction. That left ordinary taxable income at $14,945 instead of $14,316.66, overstating the tax by about $75."
-us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction instead of the 2026 OBBBA amount of $16,100 and omitted the $628 qualified-overtime deduction. That overstated ordinary taxable income by about $1,328, although the $120 Saver's Credit was applied correctly."
-us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It got within $1 through offsetting errors: it subtracted the $1,800 of health premiums from wages, used a $15,000 standard deduction and the 2025 $11,925 bracket, and never applied the $628 overtime deduction. Its unrounded $119.90 credit gave $1,351 instead of $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $12,225 top of the 10% bracket instead of the 2026 figures of $16,100 and $12,400, and it omitted the $628 qualified-overtime deduction. That overstated ordinary taxable income to $15,645 instead of $14,316.66."
-us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of $16,100 and skipped the $628 qualified-overtime deduction, which gave $1,645.40 before credits. After the Saver's Credit it ended at $1,525.50 instead of $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"It subtracted the $6,589 of ESI premiums from wages and assumed a pre-TCJA deduction plus personal exemption. It then used a nonexistent AOTC to bring the tax to zero, although no qualified tuition expenses are listed, so the AOTC is $0."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its stated taxable income of about $25,500 overstates the correct base. After the $16,100 standard deduction, the $628 overtime deduction and excluding the 0%-taxed dividends and capital-gain distributions, ordinary taxable income is $14,316.66, which gives $1,470 of tax less the $120 Saver's Credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It cut wages by the $6,589 of ESI premiums, which are not deducted from gross wages here. It also used a pre-TCJA standard deduction and personal exemption instead of the permanent 2026 standard deduction of $16,100 plus the $628 overtime deduction, so its taxable income was too low."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, using an $8,300 standard deduction plus a $5,300 personal exemption, which the OBBBA cancelled, and it subtracted the $6,589 of ESI premiums from wages. It also omitted both the $628 overtime deduction and the $120 Saver's Credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It reported $1,475, which is roughly the $1,470 pre-credit tax. It never subtracted the $120 Saver's Credit earned by $1,199 of retirement contributions at an AGI in the 10% tier."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It subtracted the $6,589 of ESI premiums from wages and applied a personal exemption with the pre-TCJA deduction, even though the 2026 rules keep the $16,100 standard deduction with no exemption. It also missed the $628 overtime deduction and the $120 Saver's Credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its reasoning reached $1,756 without the $628 overtime deduction or the $120 Saver's Credit, yet it submitted $1,086, which matches the ESI-subtraction and TCJA-sunset approach. Neither figure applies the overtime deduction and Saver's Credit that give $1,470 less $120."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction and the $11,925 bracket instead of the 2026 figures of $16,100 and $12,400, and omitted the $628 overtime deduction. Its reasoning computed $1,567, but it submitted $1,617."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"It correctly reached $1,470.04 of tax on $14,317 of ordinary taxable income. It then stopped without subtracting the nonrefundable Saver's Credit of 10% × $1,199 = $120, which brings the result to $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction and a $12,200 top of the 10% bracket instead of the 2026 figures of $16,100 and $12,400, and skipped the $628 qualified-overtime deduction. It applied the $119.90 Saver's Credit correctly but to an inflated $1,633.40 of tax."
-us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used a $15,200 standard deduction instead of $16,100 and omitted the $628 overtime deduction. It also said no nonrefundable credits apply, missing the $120 Saver's Credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The standard deduction does not eliminate taxable income: about $36,183 of AGI less $16,100 and $628 leaves about $19,455. It also relied on an AOTC that is $0 because no qualified tuition expenses are listed."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It treated the AOTC as removing the remaining liability, but no qualified tuition expenses are listed, so the AOTC is $0. Ordinary taxable income of $14,316.66 produces $1,470 of tax, offset only by the $120 Saver's Credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It computed the $1,470 pre-credit tax correctly, including the $628 overtime deduction. It then concluded that no nonrefundable credits are usable and missed the $120 Saver's Credit on $1,199 of retirement contributions."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It claimed an AOTC with no qualified tuition expenses on record, so the AOTC is $0. The liability is $1,470 less the $120 Saver's Credit, which is $1,350, not zero."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Its $1,610 equals $1,470 plus 12% of the $1,170 in capital-gain distributions, so it taxed those gains at ordinary rates instead of keeping them out of ordinary taxable income. It also left out the $120 Saver's Credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,589 of ESI premiums from wages, bringing AGI down to $29,594 and ordinary taxable income to $7,728. Wages are reduced only by the $926 traditional 401(k), which leaves $14,316.66 of ordinary taxable income."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It computed the Saver's Credit as $96.90, which leaves out the $163 Roth 401(k) and $67 Roth IRA contributions. Qualifying contributions total $1,199, so the credit is $120."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It reached the correct $1,470.04 of tax on $14,317 of ordinary taxable income. It then omitted the nonrefundable Saver's Credit of 10% of $1,199, about $120."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,589 of ESI premiums from wages, which gave an AGI of $29,594 and only $7,728 of ordinary taxable income. It also said no usable credits apply, missing the $120 Saver's Credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,"It reached the correct $1,470.04 pre-credit tax, but it did not apply the $120 nonrefundable Saver's Credit on $1,199 of 401(k) and IRA contributions."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The standard and overtime deductions leave about $19,455 of taxable income, and the AOTC is $0 because no qualified tuition expenses are listed. Only the $120 Saver's Credit reduces the $1,470 of tax."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA sunset in 2026, applying a personal exemption and the restored 15% bracket. The OBBBA made the TCJA rates and the $16,100 standard deduction permanent. It also missed the $628 overtime deduction."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied post-sunset rules ($8,300 standard deduction, $5,300 exemption, 15% bracket) instead of the permanent TCJA rules in force for 2026 ($16,100 standard deduction, 10%/12% brackets). It also omitted the $628 overtime deduction."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It reached the correct $14,317 of ordinary taxable income, but it ended the 10% bracket at $12,200 instead of the 2026 figure of $12,400, adding $4. It also subtracted a $119.90 Saver's Credit instead of the reference's $120, giving $1,354.14."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It subtracted the $6,589 of ESI premiums from wages and assumed a TCJA-sunset $8,350 standard deduction plus a $5,330 exemption. It also omitted the $628 overtime deduction and the $120 Saver's Credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It assumed the TCJA sunset, applying an $8,300 standard deduction, a $5,200 personal exemption and a 15% bracket instead of the 2026 $16,100 deduction and 10%/12% brackets. It also missed the $628 overtime deduction and understated the Saver's Credit at $97 instead of $120."
-us,scenario_091,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no value or reasoning for federal_income_tax_before_refundable_credits.
-us,scenario_091,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It claimed an AOTC with no qualified tuition expenses on record, so the AOTC is $0. The $1,470 of tax is reduced only by the $120 Saver's Credit, leaving $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used the 2025 figures of a $15,750 standard deduction and an $11,925 10% bracket instead of the 2026 figures of $16,100 and $12,400. It also taxed the $1,170 of gains at ordinary rates and omitted both the $628 overtime deduction and the $120 Saver's Credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It applied the 50% Saver's Credit rate, but an AGI of about $36,183 falls in the 10% tier, since the 2026 single 50% tier ends near $24,250. It also left the Roth contributions out of the $1,199 base and missed the $628 overtime deduction."
-us,scenario_091,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It invented a $1,960 AOTC with no qualified tuition expenses on record and used it to zero out the tax. It also added the $628 overtime premium on top of wages that already include it, instead of deducting it as qualified overtime."
-us,scenario_091,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It said the standard deduction reduces taxable income to $0, but $16,100 is far less than its own $40,571 of income. Correct taxable income is about $19,455, which gives $1,470 of tax less the $120 Saver's Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It left out the qualified-overtime deduction for the $628 FLSA premium. Its own math therefore put ordinary taxable income at $14,945 instead of $14,317 and tax at $1,425 after the $120 saver's credit. It then submitted $1,276, which none of its own steps produce."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It left out the $628 qualified-overtime deduction and taxed the $1,170 non-Schedule-D capital gains at ordinary rates instead of the 0% band. That raised ordinary taxable income to $16,115 instead of $14,317, so pre-credit tax came to $1,686 rather than $1,470 before the correct $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It reached the correct $18,285 taxable income with the $1,170 gains left out of AGI, then subtracted those same gains again as preferential income. That double exclusion cut ordinary taxable income to $13,147 instead of $14,317 and tax to $1,329.64 instead of $1,470."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It claimed a $2,500 AOTC even though no qualified tuition expenses are listed. It also taxed qualified dividends at 15% when all taxable income sits in the 0% capital-gains band, used the 2024 $14,600 standard deduction, and ignored both the $926 401(k) exclusion and the $628 overtime deduction. Its final $650 does not follow from any of its steps."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It used the nonrefundable part of the AOTC to wipe out the liability, but no qualified tuition expenses are listed, so the AOTC is $0. Ordinary taxable income of $14,317 produces $1,470 of tax, and the only nonrefundable credit is the $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It left out the $628 overtime deduction, taxed the $1,170 non-Schedule-D gains as ordinary income, and said no nonrefundable credits apply, missing the 10% saver's credit on $1,199 of contributions (≈$120). Its submitted $1,656 does not even match its own $1,686 tentative tax."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It assumed the $27,000 wages were already net of the traditional 401(k), so it never excluded the $926, and it left out the $628 overtime deduction. It then subtracted an arbitrary $331 AOTC on assumed tuition instead of the $120 saver's credit the facts support."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It left out both the $628 overtime deduction and the $120 saver's credit, then inflated its own bracket result of about $1,545 to an unsupported $2,109. With the overtime deduction, ordinary taxable income is $14,317, which gives $1,470 before the saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used 2025-era figures (a $15,000 standard deduction and an $11,925 top for the 10% bracket) instead of the 2026 $16,100 and $12,400. It also left out the $628 overtime deduction and applied no saver's credit, even though $1,199 of retirement contributions earn a 10% credit (≈$120)."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"Its $1,470 matches the correct tax before credits, but it got there by adjusting a calculation built on a $1,500 AOTC, and the AOTC is $0 with no tuition listed. It also explicitly dismissed the saver's credit; 10% of $1,199 in contributions (≈$120) brings tax to $1,350."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It left out the $628 FLSA overtime deduction, so ordinary taxable income stayed at $14,945 instead of $14,317 and pre-credit tax was $1,545 instead of $1,470. The $120 saver's credit was applied correctly."
+us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It used a $15,400 standard deduction instead of the 2026 single amount of $16,100 and left out the $628 overtime deduction. That overstated ordinary taxable income at $15,645 versus $14,317; the $119.90 saver's credit was correct."
+us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It landed $1 off through errors that roughly cancel. It wrongly subtracted the $1,800 of other health premiums from wages, used a $15,000 standard deduction and an $11,925 10% bracket instead of $16,100 and $12,400, and left out the $628 overtime deduction. The correct path is $14,317 ordinary taxable income, $1,470 of tax, minus a $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It used a $15,400 standard deduction instead of $16,100 and a $12,225 10% bracket instead of $12,400, and it left out the $628 overtime deduction. That put ordinary taxable income at $15,645 rather than $14,317."
+us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It used a $15,300 standard deduction instead of the 2026 $16,100 and left out the $628 overtime deduction, overstating taxable income by about $1,428. The saver's credit was applied correctly."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,589 ESI premiums from wages that were already stated as gross, and it used the pre-TCJA standard deduction plus a personal exemption instead of the $16,100 2026 standard deduction. It then offset the tax with an AOTC despite no qualified tuition; only the $120 saver's credit applies against $1,470 of tax."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its taxable income of about $25,500 is too high. After the $16,100 standard deduction and the $628 overtime deduction, taxable income is $19,455, with only $14,317 at ordinary rates, giving $1,470 of tax minus a $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,589 ESI premiums from gross wages and used the pre-TCJA standard deduction and exemption instead of the 2026 $16,100 standard deduction plus the $628 overtime deduction. Its saver's credit was correct, but the understated ordinary base gave $1,016 of tax instead of $1,470."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used the pre-TCJA amounts ($8,300 standard deduction plus a $5,300 exemption) instead of the 2026 $16,100 standard deduction, and it subtracted $6,589 of ESI premiums from wages already stated as gross. It also left out the $628 overtime deduction and skipped the $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It showed no calculation. Its $1,475 is about the $1,470 pre-credit tax with no reduction for the 10% saver's credit on $1,199 of retirement contributions (≈$120)."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted $6,589 of ESI premiums from gross wages and used a standard deduction plus personal exemption instead of the 2026 $16,100 standard deduction and the $628 overtime deduction. It applied no saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation computes $1,756 from AGI of $36,183 without the $628 overtime deduction or the $120 saver's credit, yet it submitted $1,086, which contradicts that explanation. Neither figure applies the overtime deduction and saver's credit that bring tax to $1,350."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction and an $11,925 10% bracket instead of the 2026 $16,100 and $12,400, and it left out the $628 overtime deduction. Its own math gave $1,567 after the saver's credit, but it submitted $1,617."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,categorical_eligibility,False,"It correctly computed $1,470.04 of tax using the $16,100 standard deduction and the $628 overtime deduction. It never subtracted the nonrefundable saver's credit, which is 10% of $1,199 in 401(k)/IRA contributions (≈$120) at AGI of about $35k."
+us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used a $15,400 standard deduction and a $12,200 10% bracket instead of the 2026 $16,100 and $12,400, and it left out the $628 overtime deduction. The $119.90 saver's credit was correct."
+us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It used a standard deduction of about $15,200 instead of $16,100, left out the $628 overtime deduction, and said no nonrefundable credits apply. That missed the 10% saver's credit (≈$120) on $1,199 of retirement contributions."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It claimed the standard deduction and the AOTC eliminate the tax. In fact, taxable income after the $16,100 standard deduction and $628 overtime deduction is $19,455, and with no tuition listed the AOTC is $0. Ordinary taxable income of $14,317 yields $1,470, reduced only by the $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It used the AOTC to eliminate tax even though no qualified tuition expenses are listed, so the AOTC is $0. The $1,470 of tax is reduced only by the $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"It correctly reached $1,470 using the $16,100 standard deduction and the $628 overtime deduction. It then said no nonrefundable credits apply, missing the 10% saver's credit on $1,199 of traditional and Roth 401(k)/IRA contributions (≈$120)."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"It used the AOTC to offset the whole liability, but with no qualified tuition listed the AOTC is $0. Tax is $1,470 minus the $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $1,610 matches taxing the $1,170 non-Schedule-D capital gains at 12% instead of the 0% band, which gives an ordinary base of $15,487 instead of $14,317. It also applied no saver's credit despite $1,199 of eligible retirement contributions."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,589 ESI premiums from wages already stated as gross, cutting AGI to $29,594 and ordinary taxable income to $7,728 instead of $14,317. Its standard deduction, overtime deduction, and saver's credit were otherwise correct."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,other,False,"Its $1,470.04 pre-credit tax is correct, but it computed the saver's credit as 10% of only the $969 in traditional contributions ($96.90). The $163 Roth 401(k) and $67 Roth IRA contributions also qualify, so the credit is 10% of $1,199 (≈$120)."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,categorical_eligibility,False,"It correctly computed $1,470.04 of tax but left out the nonrefundable saver's credit, which is 10% of $1,199 in 401(k)/IRA contributions (≈$120). Subtracting it gives $1,350."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It subtracted $6,589 of ESI premiums from wages already stated as gross, shrinking ordinary taxable income to $7,728 instead of $14,317. It also left out the $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,categorical_eligibility,False,"It correctly computed $1,470.04 of tax after the standard and overtime deductions. It never applied the nonrefundable 10% saver's credit on $1,199 of retirement contributions (≈$120)."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"It said the AOTC and other nonrefundable credits wipe out the tax, but with no tuition listed the AOTC is $0. After the $16,100 standard deduction and $628 overtime deduction, $1,470 of tax remains, reduced only by the $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA rules (a personal exemption and a 15% bracket) instead of the 2026 rules, which use a $16,100 standard deduction and a 12% bracket. It also left out the $628 overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used the pre-TCJA amounts (about $8,300 standard deduction plus a $5,300 exemption, and a 15% bracket) instead of the 2026 $16,100 standard deduction and 12% bracket. It also left out the $628 overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"Every deduction and the $119.90 saver's credit were correct, but it ended the 10% bracket at $12,200 instead of the 2026 single figure of $12,400. That added $4 of tax ($1,474.04 instead of $1,470)."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted $6,589 of ESI premiums from wages already stated as gross, and used pre-TCJA amounts ($8,350 standard deduction plus $5,330 exemption) instead of the 2026 $16,100 standard deduction. It also left out the $628 overtime deduction and the $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA rules (about $8,300 standard deduction, a $5,200 exemption, and a 15% bracket) instead of the 2026 $16,100 standard deduction and 12% bracket, and it left out the $628 overtime deduction. It also computed the saver's credit on $969 instead of $1,199, leaving out the Roth contributions."
+us,scenario_091,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value for this variable. The correct path is $14,317 of ordinary taxable income, $1,470 of tax, minus a $120 saver's credit, for $1,350."
+us,scenario_091,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"It used the AOTC to fully offset the tax even though no qualified tuition expenses are listed, so the AOTC is $0. Tax is $1,470 reduced only by the $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It used a $15,750 standard deduction and an $11,925 10% bracket instead of the 2026 $16,100 and $12,400. It also taxed the $1,170 gains as ordinary income, left out the $628 overtime deduction, and applied no saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It applied the 50% saver's credit rate to only $969, but AGI of about $35k puts a single filer in the 10% tier, and Roth contributions count, so the credit is 10% of $1,199 (≈$120). It also taxed the $1,170 gains as ordinary income and left out the $628 overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It added the $628 overtime premium on top of $27,000 wages that already include overtime, and it skipped both the $926 401(k) exclusion and the overtime deduction. It then used an AOTC with no qualified tuition to zero out the tax; only the $120 saver's credit applies against $1,470."
+us,scenario_091,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its reasoning contradicts itself: it states taxable income of $40,571, then says the standard deduction reduces it to zero. The $16,100 standard deduction and $628 overtime deduction leave $19,455 taxable ($14,317 at ordinary rates), which produces $1,470 of tax before the $120 saver's credit."
us,scenario_091,federal_refundable_credits,claude-fable-5,llm_error,other,False,"Its own reasoning reached the correct result — no qualified tuition expenses are listed, so under the prompt's unlisted-inputs-are-zero rule the American Opportunity Credit and its 40% refundable portion are $0 — and it then overrode that derivation to submit $400 as a 'minimal refundable AOTC assumption'. It also mis-stated the EITC bar as the AGI phase-out ceiling while never applying the age-25 floor that independently zeroes the childless EITC for this 22-year-old."
us,scenario_091,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It treated the AOTC administrative flags (1098-T, institution EIN, half-time enrollment, credential pursuit) as if they established up to $4,000 of qualified expenses, when tuition is an unlisted input equal to $0, so the credit computed as 100% of the first $2,000 plus 25% of the next $2,000 is $0 and the refundable portion is $0. It then submitted $1,200, exceeding the $1,000 statutory refundable cap it had itself just cited."
us,scenario_091,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"It jumped from AOTC eligibility flags straight to the $2,500 statutory maximum and took 40% of it, never computing the credit base: qualified tuition expenses are $0 here, so the credit and its refundable portion are $0. Its EITC rationale is also wrong on its own numbers — $10,152 of investment income is below the disqualifying limit — and it missed the actual disqualifier, the age-25 floor for childless EITC claimants."
@@ -8122,75 +8207,77 @@ us,scenario_091,federal_refundable_credits,kimi-k2.6,parse_contract_failure,miss
us,scenario_091,federal_refundable_credits,minimax-m3,llm_error,other,False,"It quoted only the statutory ceiling — 40% of a $2,500 maximum, capped at $1,000 — and submitted that ceiling as the answer, never computing the credit from qualified tuition expenses, which are unlisted and therefore $0, making the American Opportunity Credit and its refundable portion $0."
us,scenario_091,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"It verified every AOTC administrative condition (half-time, credential, 1098-T) and then equated eligibility with the maximum credit, taking $1,000 as the 40% refundable portion; qualified tuition expenses are $0, so the credit base is $0 and nothing is refundable. Its EITC analysis correctly found investment income under the cap but rested the zero on the AGI phase-out while missing the age-25 floor that disqualifies a 22-year-old childless filer."
us,scenario_091,head_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,The model incorrectly placed MAGI of 2.19 times FPL below Wisconsin's Medicaid income threshold for a single adult. It also treated that asserted income threshold as sufficient despite the head qualifying through no Medicaid eligibility category.
-us,scenario_091,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model got FICA right ($1,674.00 + $391.50) but then added a made-up Wisconsin 'state payroll tax' of $528.18. That figure is really state income tax withholding. Wisconsin has no mandatory employee state payroll tax (no SDI or paid-leave contribution), so the total should stay at $2,065.50."
-us,scenario_091,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $1,800 of other health insurance premiums from wages and taxed only $25,200. Those premiums are paid outside payroll, not withheld pre-tax through a cafeteria plan, so they do not reduce FICA wages. The 7.65% rate applies to the full $27,000."
-us,scenario_091,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the $6,589 of employer-sponsored insurance premiums as a pre-tax employee payroll deduction and taxed only $20,411 at 7.65%. Those premiums do not reduce FICA wages, so the base is the full $27,000 and the tax is $2,065.50."
-us,scenario_091,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the $6,589 of employer-sponsored insurance premiums from gross wages and taxed $20,411. The ESI premium input does not reduce the Social Security or Medicare wage base, which stays at $27,000 and gives $2,065.50."
-us,scenario_091,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model correctly kept the traditional 401(k) contributions in FICA wages, but it wrongly subtracted the $6,589 of employer-sponsored insurance premiums as a pre-tax deduction. That shrank the base to $20,411 instead of the full $27,000."
-us,scenario_091,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model used the right base ($27,000) but added the rates wrong: it combined 6.2% and 1.45% as 7.5% instead of 7.65%. That gives $2,025 instead of $2,065.50."
-us,scenario_091,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model defined Medicare/Social Security wages as $27,000 minus $6,589 of 'pre-tax health insurance premiums' and computed $1,265.48 + $295.96. The employer-sponsored insurance premiums do not reduce FICA wages, so each rate applies to the full $27,000."
-us,scenario_091,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model's explanation derives $2,065.50 correctly, but the value it submitted is 1,561.44. That is exactly 7.65% of $20,411, the base left after wrongly subtracting the $6,589 ESI premiums. The submitted value contradicts the explanation and carries the ESI-subtraction error."
-us,scenario_091,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"The model's explanation totals $2,065.50 on $27,000 of wages, but the value it submitted is 1,561.44. That equals 7.65% of $20,411, meaning it subtracted the $6,589 employer-sponsored insurance premiums from the FICA base. The submitted value conflicts with the explanation and contains the base error."
-us,scenario_091,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct method (6.2% + 1.45% on $27,000, no Additional Medicare Tax, no Wisconsin payroll tax) but got the arithmetic wrong. It reported $2,063.10 instead of $1,674.00 + $391.50 = $2,065.50."
-us,scenario_091,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model reached about $2,066 using 7.65% on $27,000, then raised it to $2,118 with an invented 'rounded per annual payroll-tax estimation' step. No rule supports that adjustment; the correct total is $2,065.50."
-us,scenario_091,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model listed the correct components ($1,674 + $391.50) but submitted $2,457. That equals $27,000 × 9.1%, meaning it applied the combined 2.9% Medicare rate (employee plus employer) instead of the 1.45% employee rate. The employee-only Medicare tax is $391.50, for a total of $2,065.50."
-us,scenario_091,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model explicitly assumed the employer-sponsored insurance premiums were pre-tax employee premiums and dropped the FICA base to $20,411. That exclusion does not apply: Social Security and Medicare tax apply to the full $27,000 of gross wages, giving $2,065.50."
-us,scenario_091,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model treated the $6,589 of employer-sponsored insurance premiums as a pre-tax deduction and taxed $20,411 of 'estimated' Social Security and Medicare wages. The ESI premiums do not reduce FICA wages, so the 7.65% applies to $27,000."
-us,scenario_091,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model set the FICA base at $27,000 minus $6,589 of 'pre-tax health premiums', or $20,411, and taxed that. The employer-sponsored insurance premiums are not excluded from FICA wages, so the base is $27,000 and the tax is $2,065.50."
-us,scenario_091,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value and no explanation, so there was nothing to score. The correct answer is 7.65% of $27,000 in wages, or $2,065.50."
-us,scenario_091,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model added the $628 FLSA overtime premium on top of the $27,000 gross wages, taxing $27,628. The prompt states that gross wages already include overtime pay, so this counts the overtime twice. The correct base is $27,000, giving $2,065.50."
+us,scenario_091,payroll_tax,claude-fable-5.2,llm_error,payroll_tax_base,False,"The model subtracted the $926 traditional 401(k) deferral and used a FICA base of $26,074. Elective 401(k) deferrals stay in Social Security and Medicare wages, so the base is the full $27,000 and the tax is $2,065.50."
+us,scenario_091,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model got the FICA amounts right ($1,674.00 + $391.50) but added an invented $528.18 'Wisconsin state payroll tax' based on income-tax withholding. Wisconsin has no mandatory employee payroll tax, and income-tax withholding is not a payroll tax, so the total is $2,065.50."
+us,scenario_091,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model took the $1,800 of other health insurance premiums out of FICA wages and used a base of $25,200. Those premiums do not reduce the payroll-tax base, which is the full $27,000 of gross wages, so the tax is $2,065.50."
+us,scenario_091,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the $6,589 of employer-sponsored insurance premiums as a pre-tax cafeteria-plan exclusion and used a FICA base of $20,411. PolicyEngine taxes the full $27,000 of gross wages at 7.65%, giving $2,065.50."
+us,scenario_091,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model subtracted the $6,589 of ESI premiums from wages and applied 7.65% to $20,411. The payroll-tax base is the full $27,000 of gross wages, so the employee tax is $1,674.00 + $391.50 = $2,065.50."
+us,scenario_091,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model correctly kept the 401(k) deferral in FICA wages, but it took the $6,589 of ESI premiums out of the base and got $20,411. The base is the full $27,000, so the tax is $2,065.50."
+us,scenario_091,payroll_tax,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model used a combined rate of 7.5% instead of 7.65% (6.2% + 1.45%) and got $2,025. At 7.65% on $27,000, the employee tax is $2,065.50."
+us,scenario_091,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model treated the $6,589 of ESI premiums as pre-tax and applied 6.2% and 1.45% to $20,411. The FICA base is the full $27,000 of gross wages, giving $1,674.00 + $391.50 = $2,065.50."
+us,scenario_091,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model's explanation correctly computed $1,674.00 + $391.50 = $2,065.50 on $27,000, but it submitted 1561.44 in the value field. That number matches the wrong base of $27,000 minus the $6,589 of ESI premiums, so the scored answer reflects that reduced base."
+us,scenario_091,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"The model's explanation gave the correct $2,065.50 on $27,000 of wages, but the submitted value was 1561.44. That number matches taxing only $20,411 after subtracting the $6,589 of ESI premiums, which contradicts the model's own correct derivation."
+us,scenario_091,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model set up the right base and rates (6.2% and 1.45% on $27,000) but made an arithmetic error and reported $2,063.10 instead of $2,065.50. Those inputs give $1,674.00 + $391.50 = $2,065.50."
+us,scenario_091,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly reached about $2,066 ($1,674 + $392) and then arbitrarily 'rounded' it to $2,118. No adjustment applies, and the employee payroll tax is exactly $2,065.50."
+us,scenario_091,payroll_tax,gpt-5.6-terra,llm_error,other,False,"The model listed the correct components, $1,674 for Social Security and $391.50 for Medicare, but submitted $2,457, which is 9.1% of $27,000 rather than the sum of its own figures. Those components add to $2,065.50."
+us,scenario_091,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"The model assumed the $6,589 of employer-sponsored premiums were pre-tax employee contributions and taxed only $20,411. The FICA base is the full $27,000 of gross wages, so the tax is $2,065.50."
+us,scenario_091,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"The model cut Social Security and Medicare wages to $20,411 by subtracting the $6,589 of ESI premiums. The payroll-tax base is the full $27,000, so 7.65% gives $2,065.50."
+us,scenario_091,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model subtracted the $6,589 of health premiums and used a FICA base of $20,411, which gave about $1,561. The full $27,000 of gross wages is subject to payroll tax, so the result is $1,674.00 + $391.50 = $2,065.50."
+us,scenario_091,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no usable payroll_tax value or explanation, so no answer could be scored. The correct value is 7.65% of $27,000 in gross wages, which is $2,065.50."
+us,scenario_091,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model added the $628 FLSA overtime premium on top of $27,000 of gross wages and got a FICA base of $27,628. The prompt states that gross wages already include overtime pay, so the base is $27,000 and the tax is $2,065.50."
us,scenario_091,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its reasoning used a standard deduction of about $10,668 (base $12,760, phase-out starting at $18,400), well below the engine's $12,074.61, and never subtracted the $700 exemption. That produced roughly $975. It then submitted $1,257, which contradicts its own computation."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It taxed the full federal AGI of $36,183 without the 30% exclusion on the $1,170 capital gain distributions ($351), and it computed taxable income as AGI minus the standard deduction only, leaving out the $700 exemption. Its taxable income of $24,184 is about $1,127 above the correct $23,056.95, so its tax of $932 is too high."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It left the $10,152 of interest, dividends and capital gains out of AGI and started from federal taxable income after the federal standard deduction instead of Wisconsin's own sliding-scale deduction. It then applied the federal American Opportunity Credit against Wisconsin tax, which has no AOTC, so a $878.52 liability became zero."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It claimed Wisconsin taxable income was low enough to produce zero tax. In fact Wisconsin AGI of $35,831.56, minus the $12,074.61 standard deduction and the $700 exemption, leaves $23,056.95 of taxable income. That is taxed at 3.5%/4.4% for $878.52, and no Wisconsin nonrefundable credit offsets it."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It refused the 30% exclusion on the non-Schedule-D gains, but capital gain distributions are long-term gains that receive it. It also put the phased standard deduction at about $9,500 instead of $12,074.61 and used a $14,320 first bracket. That gave $1,014, which it then raised without basis to $1,140."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It put the single standard deduction at about $7,700 instead of $12,074.61 and skipped the $351 capital gain exclusion, which inflated taxable income to $27,783. It then added unspecified 'adjustments' to raise its own $1,093 figure to $1,163."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It started from the full federal AGI without the 30% capital gain distribution exclusion and used a phased standard deduction of about $9,000 instead of $12,074.61. That left taxable income near $26,500 instead of $23,056.95, so it got $1,140."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Its structure matches the engine: Wisconsin AGI of $35,832, the phased standard deduction, the $700 exemption, and the 3.5%/4.4% rates. But it estimated the indexed 2026 standard deduction at $12,003 instead of $12,074.61, which made taxable income $23,129 instead of $23,056.95 and pushed the tax $3.48 above $878.52."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It treated capital gains as fully taxable and skipped the 30% exclusion of the $1,170 distributions. It also applied a 2024 standard deduction base of $12,230 phased out from $15,000, which gave $9,688 instead of $12,074.61, together with the 2024 $13,810 first bracket. Taxable income came out at $25,795 and the tax at $1,010.69."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It skipped the 30% capital gain exclusion and used a standard deduction of $11,340 instead of $12,074.61, which gave $933. It then subtracted an invented Wisconsin 'Married/Single credit' to reach $900, although no Wisconsin nonrefundable credit applies to this single filer."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It started from federal AGI of $36,183 without subtracting the $351 (30%) capital gain distribution exclusion and put the phased standard deduction at about $11,600 instead of $12,074.61. Its taxable income of about $23,900 is roughly $850 too high, which gives $920."
-us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It applied a flat $13,450 single standard deduction without Wisconsin's sliding-scale phase-out, which reduces the deduction to $12,074.61 at this income. It also skipped the 30% capital gain exclusion and used a $14,320 first bracket. That understated taxable income at $22,033 and gave $840.57."
-us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used AGI of $34,383 (federal AGI minus an unsupported $1,800) and an unreduced $13,230 standard deduction with no phase-out. It also left out the $700 exemption and taxed at the old 3.54%/4.65% rates instead of the current 3.5%/4.4%."
-us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It subtracted an unreduced $14,050 standard deduction with no income-based phase-out, so it overstated the deduction relative to $12,074.61. It also left out both the $700 exemption and the $351 capital gain exclusion, which gave taxable income of $22,133 and tax of $837.05."
-us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It phased the standard deduction down to $9,963.40 instead of $12,074.61, did not apply the 30% capital gain exclusion, and left out the $700 exemption. Its taxable income was $26,219.60 instead of $23,056.95, so it got $1,054.49."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It started from AGI of $29,594, which means it subtracted the $6,589 employer-sponsored insurance premiums from income. Only the $926.16 pre-tax retirement contribution reduces wages, and federal AGI is $36,182.56. It also skipped the 30% capital gain exclusion, so taxable income came out at $17,484 and the tax at $640.44."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It treated Wisconsin income as the $27,000 of wages alone. That ignores the $4,806 of interest, $4,176 of dividends and $1,170 of capital gain distributions that bring Wisconsin AGI to $35,831.56. It never showed the standard deduction, exemption or bracket steps behind its $905."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $588 implies Wisconsin taxable income of about $16,400 instead of $23,056.95, which is consistent with the same error its sibling models made: subtracting the $6,589 employer-sponsored insurance premiums from AGI. The correct path is Wisconsin AGI of $35,831.56 minus $12,074.61 and $700."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used a state AGI of $29,594, which comes from wrongly subtracting the $6,589 employer-sponsored insurance premiums. It also left out the $700 exemption and the 30% capital gain exclusion, which gave taxable income of $16,887 and tax of $614.15."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $1,177 implies Wisconsin taxable income near $29,800, about $6,700 above the correct $23,056.95. That points to a heavily understated standard deduction instead of the $12,074.61 sliding-scale amount plus the $700 exemption."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It applied a 60% long-term capital gain exclusion, but Wisconsin's is 30% ($351 here). It also subtracted a 'medical expense credit' that Wisconsin does not allow for these facts, which pulled the tax down to $543.09."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its explanation states value = 1018 but it submitted $944, and it treated the exemption as a 'personal exemption credit' instead of a $700 deduction. Both figures exceed $878.52, which shows it understated the combined deduction of $12,774.61 relative to Wisconsin AGI of $35,831.56."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $1,113 implies Wisconsin taxable income around $28,400, about $5,300 too high, which is consistent with a standard deduction far below the $12,074.61 sliding-scale amount and no 30% capital gain exclusion."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It applied the 30% capital gain exclusion and the sliding-scale deduction but never mentions the $700 exemption. Its result is exactly $50 above $878.52, which corresponds to about $1,136 of excess taxable income. In other words, its deductions came out about $1,136 below the $12,074.61 standard deduction plus $700 exemption."
-us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It claimed the Wisconsin standard deduction is fully phased out at $36,183, but at this income it is still $12,074.61. It also left out the $700 exemption and the 30% capital gain exclusion and pushed income into a 5.3% bracket, which taxed the full AGI for $1,511.27."
-us,scenario_091,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It wrongly treated the standard deduction as fully phased out, when $12,074.61 remains at this income. It also left out the $700 exemption and the $351 capital gain exclusion, so it taxed all $36,183 and got $1,460."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that Wisconsin tax is fully eliminated. But Wisconsin AGI of $35,831.56, minus the $12,074.61 standard deduction and the $700 exemption, leaves $23,056.95 taxable, which is $878.52 of tax, and no Wisconsin nonrefundable credit offsets it."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It used the federal American Opportunity Credit facts to cancel the Wisconsin tax. Wisconsin has no AOTC counterpart, so the $878.52 on $23,056.95 of taxable income stands."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $628 FLSA overtime premium and the $1,800 of other health insurance premiums to reach $33,404, but the engine subtracts neither amount from Wisconsin AGI, which is $35,831.56. Those subtractions also inflated its standard deduction to $12,513.92 and cut the tax to $752.01."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $943 implies taxable income of about $24,500, roughly $1,460 above the correct $23,056.95, which is consistent with skipping the 30% capital gain exclusion and understating the $12,074.61 standard deduction or leaving out the $700 exemption."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It followed the right steps but estimated the 2026 parameters slightly too generously. Its $847 is $31.52 below $878.52, which corresponds to about $716 too little taxable income, consistent with an overstated indexed standard deduction relative to $12,074.61."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It gave no computation. Its $860 is $18.52 below the engine's $878.52, which corresponds to about $420 too little taxable income, consistent with an overstated sliding-scale standard deduction relative to $12,074.61."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its $27,443 Wisconsin income equals $35,831.56 minus both the $1,800 of other health insurance premiums and the $6,589 employer-sponsored insurance premiums. The engine subtracts neither premium amount in this reference. Those subtractions drove taxable income down to $13,714.96 and the tax to $480.02."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It used $36,183 as Wisconsin income without the $351 exclusion (30% of capital gain distributions). Its $936 implies taxable income about $1,300 above $23,056.95, so it also understated the $12,074.61 standard deduction."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It taxed about $36,183 of income without the 30% capital gain distribution exclusion that lowers Wisconsin AGI to $35,831.56. Its $920 implies taxable income roughly $940 too high, so it also understated the standard deduction relative to $12,074.61."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its $27,443 Wisconsin income comes from subtracting the $6,589 employer-sponsored insurance premiums and the $1,800 of other health premiums from $35,831.56. The engine subtracts neither premium amount in this reference, and they shrank taxable income to $13,769.76, which gave $481.94."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It applied only the retirement deductions before the standard deduction and does not mention the 30% capital gain exclusion. Its $922.88 implies about $1,000 of excess taxable income, which is consistent with the missing $351 exclusion plus an understated standard deduction."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and credits fully offset the tax. The $12,074.61 standard deduction and $700 exemption leave $23,056.95 taxable out of $35,831.56 of Wisconsin AGI, and no Wisconsin nonrefundable credit applies."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It started from $36,183 without the $351 capital gain distribution exclusion and phased the standard deduction down to about $11,046 instead of $12,074.61. That produced taxable income of $24,437 and tax of $946."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It correctly reached $35,832 after the 30% exclusion but used a sliding-scale standard deduction of about $10,618 instead of $12,074.61, together with a $14,320 first bracket instead of about $15,110. Taxable income came out at $24,514 and the tax at $950."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It subtracted the $628 overtime premium from federal AGI to get $35,555. The federal overtime deduction is not an AGI adjustment and does not carry into Wisconsin income, so Wisconsin AGI stays at $35,831.56. Its $859 is therefore below $878.52."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used AGI of $29,594, which wrongly deducts the $6,589 employer-sponsored insurance premiums. It also skipped the 30% capital gain exclusion and the $700 exemption, which gave taxable income of $16,811 and tax of $613."
-us,scenario_091,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It started near $36,183 without the $351 capital gain exclusion and used a phased standard deduction of about $10,900 to $11,000 instead of $12,074.61. It also applied an outdated 4.65% second-bracket rate instead of 4.4%."
-us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so nothing could be scored against the $878.52 reference."
-us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It applied the correct structure (Wisconsin income of $35,832, the phased standard deduction, the $700 exemption and the 3.5%/4.4% brackets). But it estimated the indexed standard deduction at $11,868.47 instead of $12,074.61, which made taxable income $23,263.53 instead of $23,056.95 and the tax $887.12."
-us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It claimed the Wisconsin School Property Tax Credit wipes out the tax. That credit requires rent or property taxes paid, and none are listed, so no credit reduces the $878.52."
-us,scenario_091,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It took federal AGI of $36,183 minus only a standard deduction to reach about $25,000 of taxable income, without the 30% capital gain exclusion or the $700 exemption. That overstated taxable income by roughly $1,950 relative to $23,056.95."
-us,scenario_091,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It used Wisconsin taxable income of about $23,660, roughly $600 above the correct $23,056.95. That is consistent with skipping the $351 capital gain distribution exclusion (plus its knock-on effect on the phase-out) and slightly understating the $12,074.61 standard deduction."
-us,scenario_091,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used a standard deduction of about $13,480 by phasing from a made-up $27,800 threshold, which overstates it relative to $12,074.61. It also left out both the $700 exemption and the 30% capital gain exclusion, which gave taxable income of $22,700 and $863."
-us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It subtracted half of the $6,589 employer-sponsored insurance premiums from AGI and used unphased standard deductions around $13,500. It applied the old 3.54%/4.65% rates and treated the exemption as a $77 credit, then averaged several guesses to reach $720.73."
-us,scenario_091,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It overstated income at $40,571 against Wisconsin AGI of $35,831.56 and used a flat $12,200 standard deduction. It then asserted that unnamed Wisconsin nonrefundable credits cancel its own $1,374.94 of tax, although no Wisconsin nonrefundable credit applies to this filer."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It phased the standard deduction down from a $12,760 base above $18,400, which gave about $10,668 instead of $12,074.61. It also never subtracted the $700 exemption, putting taxable income at about $25,164 instead of $23,056.95. It then submitted $1,257 even though its own recomputation gave about $975."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It used $36,183 as Wisconsin income without the 30% ($351) capital gain exclusion and left out the $700 personal exemption. It also used a 2025-era first bracket of $14,680 instead of about $15,110. Taxable income came out at $24,184 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,state_local_rule,False,"It excluded the entire $1,170 of capital gain distributions, giving Wisconsin AGI of $35,013, when only 30% ($351) is subtracted for AGI of $35,831.56. It also used 2025 deduction and bracket parameters, so its standard deduction was $11,704 instead of $12,074.61 and its first bracket was $14,680 instead of about $15,110."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It left out the $4,806 of interest, $4,176 of dividends and $1,170 of capital gains, and started from federal taxable income instead of Wisconsin AGI. It then treated the federal American Opportunity Credit as a Wisconsin nonrefundable credit that wipes out the tax. Wisconsin has no AOTC analog, so the $878.52 of tax on $23,056.95 of taxable income stands."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It claimed the standard deduction and deferrals leave no Wisconsin tax, but income here is $35,831.56. After the $12,074.61 sliding-scale deduction and the $700 exemption, $23,056.95 remains taxable and produces $878.52 of tax. No nonrefundable credit offsets it."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It wrongly treated the non-Schedule D capital gain distributions as ineligible for Wisconsin's 30% long-term gain exclusion and so skipped the $351 subtraction. It also put the phased standard deduction at about $9,500 instead of $12,074.61. It then submitted $1,140 even though its own arithmetic gave $1,014."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It put the phased-down single standard deduction at about $7,700, far below the correct $12,074.61 at this income, and skipped the $351 capital gain exclusion. Taxable income came out at $27,783 instead of $23,056.95. It then added an unexplained $70 on top of its $1,093 result."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It phased the standard deduction down to about $9,000 instead of $12,074.61 and skipped the 30% ($351) capital gain exclusion. Taxable income came out at about $26,500 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It followed the correct steps: the $351 exclusion, the sliding deduction, the $700 exemption and 3.5%/4.4% rates. But its estimate of the indexed 2026 phased standard deduction was $12,003 instead of $12,074.61. That overstated taxable income by about $72 ($23,129 vs $23,056.95) and gave $882 instead of $878.52."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It skipped the 30% capital gain exclusion and applied 2024 parameters: a $12,230 deduction base phasing out from $15,000 and a $13,810 first bracket. That gave a standard deduction of $9,688 instead of the 2026 phased amount of $12,074.61, so taxable income was overstated by about $2,700."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It skipped the $351 capital gain exclusion, understated the phased deduction ($11,340 vs $12,074.61) and used a $14,320 first bracket. It then subtracted about $33 for 'Married/Single' and personal exemption credits that do not apply to this filer. Its $24,143 of taxable income exceeds the correct $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It skipped the 30% ($351) capital gain exclusion and put the phased standard deduction at about $11,600 instead of $12,074.61. Taxable income came out at about $23,900 instead of $23,056.95, and it used a $14,700 first bracket instead of about $15,110."
+us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It used an unphased $13,450 standard deduction and never applied the 12% sliding-scale phase-down, which cuts it to $12,074.61 at $35,831.56 of income. It also skipped the $351 capital gain exclusion and used a $14,320 first bracket. Taxable income came out at $22,033 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It understated AGI at $34,383 instead of $36,182.56, used an unphased $13,230 standard deduction and left out the $700 exemption. It also applied the superseded 3.54%/4.65% rates instead of the 3.5%/4.4% rates now in effect."
+us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It subtracted a flat $14,050 standard deduction without the sliding-scale phase-down to $12,074.61. It also left out both the $700 exemption and the $351 capital gain exclusion, so taxable income came out at $22,133 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It phased the standard deduction down to $9,963 instead of $12,074.61, skipped the $351 capital gain exclusion and dropped the $700 exemption. Taxable income came out at $26,219.60 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,589 of employer-sponsored insurance premiums from income, giving AGI of $29,594. Those premiums are not a reduction of reported wages here; federal AGI is $36,182.56. That cut taxable income to $17,484 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It taxed $27,000 of 'adjusted gross income', which ignores $10,152 of interest, dividends and capital gains and the $926 401(k) deferral. It never built the $35,831.56 of Wisconsin AGI that, after the $12,074.61 deduction and $700 exemption, gives $878.52."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its $588 implies Wisconsin taxable income of roughly $16,000–17,000, about $6,500 below the correct $23,056.95. That gap is consistent with subtracting the $6,589 of employer-sponsored insurance premiums from wages, which does not reduce AGI here."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,589 of employer-sponsored insurance premiums to get AGI of $29,594 instead of $36,182.56. It also dropped the $700 exemption and skipped the $351 capital gain exclusion, so taxable income came out at $16,887 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Its $1,177 implies Wisconsin taxable income of about $29,800, roughly $6,800 above the correct $23,056.95. That is consistent with phasing the sliding-scale standard deduction down far below $12,074.61 and skipping the $351 capital gain exclusion."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It applied a 60% exclusion, which Wisconsin reserves for farm assets, instead of the 30% exclusion for these capital gain distributions. It also subtracted a medical expense credit that does not reduce tax here. That cut the tax to $543.09 instead of $878.52 on $23,056.95 of taxable income."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Its explanation says $1,018 while it submitted $944, and it shows no actual deduction or bracket computation. The correct steps are the $351 capital gain exclusion, the $12,074.61 phased deduction and the $700 exemption, giving $23,056.95 of taxable income and $878.52 of tax. Both of its figures imply taxable income at least $1,500 too high."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Its $1,113 implies taxable income of about $28,400, roughly $5,300 above the correct $23,056.95. That is consistent with phasing the standard deduction down well below $12,074.61 and skipping the $351 capital gain exclusion."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"It named the correct steps, the 30% exclusion, sliding-scale deduction and 3.5%/4.4% brackets, but its result is exactly $50 too high. That corresponds to taxable income about $1,136 above $23,056.95, i.e. a phased standard deduction of about $10,940 instead of $12,074.61."
+us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It wrongly treated the single standard deduction as fully phased out at $36,183 of AGI. At $35,831.56 of Wisconsin AGI it is still $12,074.61. It also dropped the $700 exemption and the $351 capital gain exclusion, and taxed part of the income at 5.3%."
+us,scenario_091,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It wrongly treated the standard deduction as fully phased out and taxed all $36,183 of federal AGI. The single sliding-scale deduction is still $12,074.61 at this income, and the $700 exemption and $351 capital gain exclusion also apply, leaving only $23,056.95 taxable."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It claimed state treatment eliminates the tax, but Wisconsin taxable income is $23,056.95 after the $12,074.61 deduction and $700 exemption. That produces $878.52, and no nonrefundable credit offsets it."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It applied the federal American Opportunity Credit against Wisconsin tax. Wisconsin has no AOTC nonrefundable credit, so the $878.52 owed on $23,056.95 of taxable income stays in place."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It cut Wisconsin income to $33,404 by subtracting the $628 FLSA overtime premium and the $1,800 of health insurance premiums on top of the $351 capital gain exclusion. Neither the overtime nor those premiums reduces Wisconsin income here; the correct figure is $35,831.56."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Its $943 implies taxable income of about $24,500, roughly $1,450 above the correct $23,056.95. That is consistent with skipping the $351 capital gain exclusion and phasing the deduction down to about $11,000 instead of $12,074.61."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It named the correct deduction, exemption and capital gain subtraction, but its $847 implies taxable income of about $22,340, roughly $715 below $23,056.95. That corresponds to overstating the phased standard deduction at about $12,790 instead of $12,074.61."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $860 implies taxable income of about $22,630, roughly $420 below the correct $23,056.95. That is consistent with overstating the 2026 sliding-scale standard deduction above $12,074.61."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted both the $6,589 of employer-sponsored insurance premiums and the $1,800 of other health premiums, cutting Wisconsin income to $27,443. The correct figure is $35,831.56, with no premium subtraction. As a result it taxed only $13,714.96 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It taxed about $36,183 of income without the 30% ($351) capital gain exclusion. Its $936 implies taxable income of about $24,400 instead of $23,056.95, so it also understated the $12,074.61 phased standard deduction."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It started from $36,183 without subtracting the $351 capital gain exclusion. Its $920 implies taxable income of about $24,000, roughly $950 above $23,056.95, so it also understated the phased standard deduction."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It cut Wisconsin income to $27,443 by subtracting both the $6,589 of employer-sponsored insurance premiums and the $1,800 of other health premiums. The correct Wisconsin AGI is $35,831.56, so it taxed $13,769.76 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,"It reduced income only for retirement deductions and never mentioned the 30% capital gain exclusion. Its $922.88 implies taxable income of about $24,070, roughly $1,000 above $23,056.95, consistent with skipping the $351 exclusion and understating the $12,074.61 phased deduction."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It claimed the standard deduction and credits fully offset tax, but the sliding-scale deduction at this income is only $12,074.61. With the $700 exemption that leaves $23,056.95 taxable and $878.52 owed, and no nonrefundable credit applies."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It taxed federal AGI of $36,183 without the 30% ($351) capital gain exclusion. It also phased the deduction down to about $11,046 instead of $12,074.61, putting taxable income at about $24,437 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It applied the $351 exclusion correctly but phased the standard deduction down to about $10,618 instead of $12,074.61, and it used a $14,320 first bracket instead of about $15,110. Taxable income came out at $24,514 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It started from federal AGI of $35,555 by excluding the $628 FLSA overtime premium. That premium stays in wages and AGI; federal AGI is $36,182.56 and Wisconsin AGI is $35,831.56. The understated income pulled its tax down to $859."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used AGI of $29,594, which wrongly subtracts the $6,589 of employer-sponsored insurance premiums. It also skipped the $351 capital gain exclusion and the $700 exemption, so taxable income came out at $16,811 instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It phased the deduction down to about $10,900–11,000 instead of $12,074.61 and skipped the $351 capital gain exclusion, which gave about $24,500 of taxable income. It also cited the superseded 4.65% second-bracket rate instead of 4.4%."
+us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no usable value or explanation for state_income_tax_before_refundable_credits, so nothing could be scored against $878.52."
+us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It used the correct $35,832 of Wisconsin income, the $700 exemption and 3.5%/4.4% rates. But it computed the phased standard deduction as $11,868.47 instead of $12,074.61, overstating taxable income by about $206 ($23,263.53 vs $23,056.95)."
+us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It offset the tax with Wisconsin's School Property Tax Credit, which requires rent or property taxes paid. None are listed here, so the credit is zero and the $878.52 owed on $23,056.95 of taxable income stands."
+us,scenario_091,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"It taxed about $25,000 starting from $36,183, skipping the $351 capital gain exclusion and understating the phased standard deduction. The deduction is $12,074.61, and with the $700 exemption taxable income is $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"It reached about $23,660 of taxable income instead of $23,056.95, which is consistent with skipping the $351 capital gain exclusion and slightly understating the $12,074.61 phased deduction. It also used a lower first bracket (about $14,680) than the 2026 figure of about $15,110."
+us,scenario_091,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It placed the phase-out start near $27,800, which left the standard deduction at $13,480 instead of $12,074.61. It also dropped the $700 exemption and the $351 capital gain exclusion. The errors partly offset, giving $22,700 of taxable income instead of $23,056.95."
+us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It subtracted half of the employer-sponsored insurance premiums ($3,294.50) from wages and used the superseded 3.54%/4.65% rates. It also treated the $700 exemption as an $77 credit. It then 'averaged' arbitrary standard-deduction guesses instead of computing the $12,074.61 phased amount."
+us,scenario_091,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It overstated income at $40,571 and used an unphased $12,200 deduction. It then claimed unnamed Wisconsin nonrefundable credits cut the $1,374.94 tax to zero. No Wisconsin nonrefundable credit applies, so $878.52 is owed on $23,056.95 of taxable income."
us,scenario_091,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_092,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the full $17,400 Social Security benefit as gross income instead of applying the provisional-income formula, under which none of it is taxable. With AGI of $14,980, the aged-single standard deduction eliminates all taxable income."
us,scenario_092,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included 85% of Social Security by comparing total income directly with the $34,000 threshold; Social Security taxation uses provisional income, including only one-half of benefits, and yields no taxable Social Security here. It also inconsistently applied the basic and age-based standard-deduction amounts, but the correct $14,980 AGI is fully offset by the applicable deduction."
@@ -8235,127 +8322,130 @@ us,scenario_093,dependent1_medicare_eligible,claude-opus-4.7,llm_error,age_disab
us,scenario_093,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model recited the correct statutory rule — 24 months of SSDI receipt confers Medicare — then discarded it, concluding that ""for tax-benefit modeling purposes, a disabled individual is treated as Medicare-eligible."" PolicyEngine's is_medicare_eligible implements no such shortcut: it is the age >= 65 test alone, which Dependent 1 at age 27 fails."
us,scenario_093,dependent1_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model stated a disjunctive engine rule — ""In PolicyEngine, Medicare eligibility is determined by age 65 or older OR disability status"" — and fired the second branch off the is_disabled flag. That OR branch does not exist in is_medicare_eligible, which evaluates age >= 65 only, so Dependent 1 at 27 is not eligible."
us,scenario_093,dependent1_medicare_eligible,gpt-6-astra,llm_error,age_disability,False,"The model claimed eligibility ""under the modeled disability-based Medicare eligibility criterion because disability is reported,"" attributing a disability criterion to PolicyEngine that is_medicare_eligible does not implement. The only modeled criterion is age >= 65, which Dependent 1 at age 27 fails, and the reported disability flag carries no SSDI entitlement, ESRD, or ALS status on these facts."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It got AGI ($112,408), the $32,200 standard deduction and the $500 credit right. But it also subtracted a $1,000 auto-loan-interest deduction, which the engine does not allow, and that gave $8,509. It then replaced that figure with an unexplained 'parameter adjustment' to $8,067. Without the auto-loan deduction the result is $9,129 − $500 = $8,629."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to the joint return's income, which raised taxable income from $80,208 to $124,208 and pushed income into the 22% bracket. It also subtracted a $1,000 auto-loan-interest deduction that does not apply. Only Head's and Spouse's income counts toward the joint return's AGI of $112,408."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to the joint return. It also used outdated brackets and a $29,200 standard deduction instead of the 2026 values ($32,200, with the 10% bracket ending at $24,800). It then subtracted an invented $4,246 of 'refundable credits', even though this output comes before refundable credits, and it never applied the $500 credit for other dependents."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"Its own reasoning estimated about $5,000 of tax, but it submitted 0, which contradicts that. Taxable income of $80,208 after the $32,200 standard deduction produces $9,129 of tax, and the $500 credit for other dependents brings it to $8,629."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It ruled out the disabled 27-year-old as a dependent under the support test, so it dropped the $500 credit for other dependents. The engine grants the $500 credit for the listed adult tax-unit dependent. It also used a $31,500 standard deduction and a $24,150 10% bracket instead of the 2026 values of $32,200 and $24,800, which overstated tax by about $97."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It ruled out the dependent under the qualifying-relative gross-income test. The engine grants the $500 credit for the listed adult tax-unit dependent. It also used a $31,500 standard deduction and a $24,000 10% bracket instead of the 2026 values of $32,200 and $24,800."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It deducted the full $3,000 of student loan interest instead of the $2,500 cap and used a $32,600 standard deduction instead of $32,200, which gave $8,371 after the $500 credit. It then added about $2,000 for the dependent's own tax. The frozen reference covers only the joint return. The output is excluded because the prompt leaves the inclusion of the dependent's separate return unresolved."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to the joint return, which pushed taxable income to $124,208 and into the 22% bracket. It also took a $1,000 auto-loan-interest deduction that does not apply. The correct taxable income is $80,208, which gives tax of $9,129 before the $500 credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It got AGI ($112,408) right, but it used the pre-OBBBA $30,000 standard deduction and a $23,850 10% bracket instead of the 2026 values of $32,200 and $24,800. It also ruled out the disabled adult child as a dependent under the qualifying-relative income test, so it missed the $500 credit for other dependents."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It got AGI ($112,408) right but used a $30,300 standard deduction instead of $32,200. It then raised its own $9,376 figure to $10,800 without basis. It also treated the 27-year-old as not a dependent, so it missed the $500 credit for other dependents."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It got AGI and the $32,200 standard deduction right, but it subtracted a $1,000 auto-loan-interest deduction that does not apply. It also ruled the 27-year-old out as a dependent because of their earnings, which removed the $500 credit for other dependents. The engine grants the $500 credit for the listed adult tax-unit dependent."
-us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to the joint return. It also wrongly treated the IRA and student-loan deductions as fully phased out, used a $30,600 standard deduction, and omitted the $500 credit for other dependents. The correct joint AGI is $112,408, with $2,500 of student loan interest and the IRA deduction both allowed."
-us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the ESI premiums as pre-tax deductions, which understated the joint AGI, and then added a separate single-return tax of $2,354.82 for the dependent. The frozen reference covers only the joint return: AGI of $112,408 with no ESI exclusion, giving $9,129 of tax minus the $500 credit. The output is excluded because the prompt leaves the inclusion of the dependent's separate return unresolved."
-us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the $8,389 ESI premiums from each person's wages, which made parental AGI $95,630 instead of $112,408. It added the dependent's own single-return tax, and it applied no nonrefundable credits, so the $500 credit for other dependents was missing."
-us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the ESI premiums from wages, giving parental AGI of $95,630 instead of $112,408. It then added $2,309.98 of tax from a separate single return for the dependent. Adding the dependent's separate return follows the household-scope reading recorded in the adjudication, which excludes the output from scoring. The $500 credit was applied correctly."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired in 2026 and used personal exemptions with a roughly $15,850 standard deduction. OBBBA made the TCJA structure permanent, with a $32,200 MFJ standard deduction and no exemptions. It also subtracted ESI premiums and added a separate return for the dependent."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent's wages, using $163,156 of total household income on the joint return, and it deducted the full $3,000 of student loan interest instead of the $2,500 cap. It also never applied the $500 credit for other dependents. Joint AGI counts only Head's and Spouse's income: $112,408."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA rules with personal exemptions and a smaller standard deduction, but OBBBA keeps the $32,200 MFJ standard deduction and 10%/12% brackets for 2026. The frozen reference includes a $500 credit for the listed adult dependent. Adding the dependent's separate return follows the household-scope reading recorded in the adjudication, which excludes the output from scoring."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It used TCJA-sunset rules (personal exemptions and a smaller standard deduction) instead of the OBBBA 2026 values. It also subtracted ESI premiums, which made parental AGI $95,630 instead of $112,408, and it added $3,006.65 of tax from a separate single return for the dependent."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. The frozen reference is $8,628.99, and the adjudicated household-scope alternative is $11,848.99. The output is excluded from scoring because the prompt leaves that scope unresolved."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted ESI premiums as pre-tax deductions, which understated joint AGI, and then added the dependent's separate single-return tax. The frozen reference covers only the joint return with the dependent claimed: $9,129 − $500. The output is excluded because the prompt leaves the inclusion of the dependent's separate return unresolved."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It added the tax from a separate single return for the dependent to the joint return's tax. The frozen reference covers only the joint return on $80,208 of taxable income, minus the $500 credit for the dependent. The output is excluded because the prompt leaves the inclusion of the dependent's separate return unresolved."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to the joint return's income, giving AGI of about $157,598. It also subtracted personal exemptions, which are $0 under 2026 law, and omitted the $500 credit for other dependents."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,household_unit_or_filing_status,False,"It recognised the disabled 27-year-old as a qualifying child, but it then deliberately chose a 'single tax unit aggregation' that added the dependent's $45,000 of wages to AGI, making it $157,408. Only Head's and Spouse's income counts toward joint AGI, which is $112,408."
-us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed the $500 credit for the dependent and added $3,353.74 of tax from the dependent's separate return. The frozen reference covers only the joint return; the prompt leaves the inclusion of the dependent's return unresolved, so the output is excluded from scoring. It also used a $30,840 standard deduction and a $23,850 10% bracket instead of the 2026 values of $32,200 and $24,800."
-us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It treated the disabled 27-year-old as self-supporting and not a dependent, which dropped the $500 credit, and then added $3,210 of tax from a separate single return. It also deducted the full $3,000 of student loan interest instead of the $2,500 cap and used a $25,200 10% bracket instead of $24,800."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed the deductions wipe out all tax. In fact, joint AGI of $112,408 minus the $32,200 standard deduction leaves $80,208 of taxable income and $9,129 of tax, which is $8,629 after the $500 credit for other dependents."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It cited 'itemized medical' effects. The medical expenses counted by the engine fall below the 7.5%-of-AGI floor, so the $32,200 standard deduction applies. With taxable income of $80,208, the tax is $9,129, minus the $500 credit. Its submitted tax is $2,281.99 below the frozen reference."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It counted all three earners' wages on the joint return and subtracted all three $8,389 ESI premiums, which gave $133,775 of wages and AGI of $132,241. Joint AGI counts only Head's and Spouse's wages after 401(k) deferrals, with no ESI exclusion: $112,408."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to joint taxable income, giving $124,708 instead of $80,208 and pushing income into the 22% bracket. It also applied no nonrefundable credit, so it missed the $500 credit for the disabled adult dependent."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the whole household's wages, including the dependent's $45,000, on the joint return, giving taxable income of $125,398. It also left out the traditional IRA deduction. Joint taxable income is $80,208, with tax of $9,129 before the $500 credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It applied the brackets and the $500 credit correctly, but it put taxable income at $85,208 instead of $80,208. The frozen reference reduces IRS gross income of $115,098 by about $2,690 in adjustments and the $32,200 standard deduction to taxable income of $80,208."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the dependent's wages on the joint return and subtracted all three ESI premiums as pre-tax deductions, which gave AGI of $132,241. Joint AGI includes only Head's and Spouse's income, with no ESI exclusion: $112,408."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It got the joint tax of $9,128.96 right, but it omitted the $500 credit for the disabled adult dependent. It added $3,220 of tax from the dependent's separate return, following the household-scope reading recorded in the adjudication. The frozen reference covers only the joint return; the output is excluded from scoring because the prompt leaves that scope unresolved."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the $16,778 of ESI premiums from the spouses' wages, which made joint AGI $95,630 instead of $112,408 and taxable income $63,430 instead of $80,208. ESI premiums are not excluded from wages in this computation."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the whole household's income, including the dependent's $45,000 of wages, on the joint return, giving taxable income of about $125,398 and 22%-bracket tax. Joint taxable income is $80,208, and the tax is $9,129 before the $500 credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed total household wages, including the dependent's, instead of only Head's and Spouse's income. Joint AGI is $112,408 and taxable income is $80,208, giving $9,129 of tax minus the $500 credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied TCJA-sunset rules for 2026: personal exemptions, a smaller standard deduction and a 15% bracket. Under OBBBA the 2026 values are a $32,200 standard deduction and 10%/12% brackets. It also added $4,846 of tax from a separate single return for the dependent and omitted the $500 credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It used post-TCJA-sunset parameters: a $16,600 standard deduction, $5,300 exemptions and a 15% rate. OBBBA's 2026 values are a $32,200 standard deduction and 10%/12% brackets. It also added $4,130 of tax from a separate return for the dependent and omitted the $500 credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It added $3,224 of tax from a separate single return for the dependent, and it applied no nonrefundable credits, so it missed the $500 credit for other dependents. It also invented a $173 overtime deduction even though no overtime premium was listed."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It used post-TCJA 2026 brackets with a 15% rate and a $30,750 standard deduction instead of OBBBA's 10%/12% brackets and $32,200 deduction. It also added $3,820 of tax from a separate single return for the dependent and omitted the $500 credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It got the joint tax of about $9,129 right, but it did not subtract the $500 credit for the disabled adult dependent. It added about $3,220 of tax from the dependent's separate return, following the household-scope reading recorded in the adjudication. The frozen reference covers only the joint return; the output is excluded from scoring because the prompt leaves that scope unresolved."
-us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,No answer or reasoning was returned for this variable.
-us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It treated all three members as one joint return and included the dependent's $45,000 of wages, which made taxable income $129,456. It also wrongly treated the traditional IRA deduction as fully phased out. Joint taxable income is $80,208."
-us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no reasoning, and a value of 0 means it found no tax at all. Joint taxable income of $80,208 produces $9,129 of tax, which is $8,629 after the $500 credit for other dependents."
-us,scenario_093,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,household_unit_or_filing_status,False,"It got AGI ($112,408) right but used a standard deduction of about $31,000 instead of the 2026 MFJ value of $32,200. It also applied no nonrefundable credits, so it missed the $500 credit for the disabled adult dependent."
-us,scenario_093,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"It got the joint taxable income of $80,208 right, but it omitted the $500 credit for other dependents. It added about $3,225 of tax from the dependent's separate return, following the household-scope reading recorded in the adjudication. The frozen reference covers only the joint return; the output is excluded from scoring because the prompt leaves that scope unresolved."
-us,scenario_093,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to the joint return's income, giving AGI of $157,408 and 22%-bracket tax. Joint AGI includes only Head's and Spouse's income: $112,408, which leaves taxable income of $80,208."
-us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the $16,778 of ESI premiums from the spouses' wages, deducted the full $3,000 of student loan interest instead of the $2,500 cap, and left out the IRA deduction. It also ruled out the $500 credit for the adult dependent. Its final $7,625.75 does not match any of its own intermediate totals."
-us,scenario_093,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the dependent's wages, using $163,000 of total wages, and deducted the full $3,000 of student loan interest. It also said the credit for other dependents should not be subtracted. It then submitted $3,778, which contradicts its own $13,491 figure."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It got AGI ($112,408), the $32,200 standard deduction and the $500 credit right. But it took a $1,000 auto-loan-interest deduction even though the qualifying-vehicle conditions were not given and so default to false, which cut taxable income to $79,208. It then made an unexplained cut of about $442, from $8,509 down to $8,067."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to the joint return. The tax unit's gross income covers only the head's and spouse's income, so it inflated AGI to $156,408 and pushed income into the 22% bracket. It also took a $1,000 auto-loan-interest deduction that does not apply."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It included the claimed dependent's $45,000 of wages on the joint return, which raised AGI to $157,408 instead of $112,408 and put income in the 22% bracket. It also took an auto-loan-interest deduction for a vehicle that does not qualify."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's wages to the joint return and used outdated values: a $29,200 standard deduction and a 10% bracket ending at $23,500. It then subtracted an invented $4,246 for 'refundable credits' and never applied the $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"Its own reasoning estimated about $5,000 of tax, yet it submitted 0. The correct calculation leaves $80,208 taxable after the $32,200 standard deduction, giving $9,128.99 of tax before the $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It refused to claim the disabled 27-year-old by applying a support test, but a permanently disabled child is a qualifying child at any age and is eligible for the $500 credit for other dependents. It also used a $31,500 standard deduction and a $24,150 bracket edge instead of the 2026 values of $32,200 and $24,800."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It applied the qualifying-relative gross-income test to deny the dependent. The disabled child is a qualifying child at any age, so the gross-income test does not apply and the $500 credit for other dependents is allowed. It also used a $31,500 standard deduction and a 10% bracket of about $24,000 instead of $32,200 and $24,800."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It deducted the full $3,000 of student-loan interest instead of the $2,500 cap and used a $32,600 standard deduction instead of $32,200. Its own figure of $8,371 then got an arbitrary $2,000 added to represent the dependent's tax. The dependent's income is not taxed on this return."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It included the claimed dependent's $45,000 of wages in the joint return's income, which gave $157,408 of AGI and 22%-bracket tax. The tax unit's gross income covers only the head and spouse. It also took a $1,000 auto-loan-interest deduction that does not apply."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It denied dependent status under the qualifying-relative gross-income test, but a permanently disabled child of any age is a qualifying child and is eligible for the $500 credit for other dependents. It also used a $30,000 standard deduction and a 10% bracket ending at $23,850 instead of the 2026 OBBBA values of $32,200 and $24,800."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It reached the correct $112,408 AGI but used a $30,300 standard deduction instead of $32,200, giving $9,376. It then inflated that to $10,800 with no basis and never applied the $500 credit for other dependents for the disabled adult qualifying child."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It ruled out the dependent because of their $45,000 of earnings. A permanently disabled child is a qualifying child regardless of age or income, so the $500 credit for other dependents applies. It also took a $1,000 auto-loan-interest deduction for a vehicle that does not qualify, which lowered tax by $120."
+us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to the joint return. It also wrongly treated the IRA and student-loan-interest deductions as fully phased out at that inflated income, used a $30,600 standard deduction, and gave no $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It cut the wages on both returns by the ESI premiums even though wages are already the taxable box-1 amount. It then added tax from a separate single return for the dependent, who is claimed on the joint return and does not file one. No $500 credit for other dependents was applied."
+us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the $8,389 ESI premiums from each person's wages, which put the parents' AGI at $95,630 instead of $112,408. It then combined the joint return with a separate single return for the dependent and omitted the $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It reduced the parents' wages by $16,778 of ESI premiums, which gave $95,630 of AGI instead of $112,408. It then added $2,309.98 of tax from a separate return for the claimed dependent, which is not part of this tax unit's liability."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA expired in 2026, with personal exemptions and a $15,850 standard deduction, but OBBBA made the TCJA permanent with a $32,200 joint standard deduction. It also subtracted ESI premiums from wages and added a separate tax for the dependent."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It built income from all three earners ($163,156) instead of only the head's and spouse's, and it deducted the full $3,000 of student-loan interest instead of the $2,500 cap. It also gave no $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It used reverted pre-TCJA rules with personal exemptions, but OBBBA kept the TCJA brackets and set a $32,200 standard deduction for 2026. It also had the claimed dependent file separately and added that tax."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA 2026 rules with personal exemptions instead of the permanent OBBBA parameters. It cut AGI to $95,630 by subtracting ESI premiums and added a separate single-return tax for the dependent."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no reasoning. The correct path is $112,408 of AGI less the $32,200 standard deduction, which leaves $80,208 taxable, $9,128.99 of tax, and $8,628.99 after the $500 credit for other dependents. Its $11,048 is about $2,419 too high, which matches taxing roughly $20,000 of extra income at 12%, i.e. counting income from beyond the head and spouse."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the ESI premiums from wages and computed a separate return for the dependent, then added that tax to the joint return's. The correct answer is a single joint return with $112,408 of AGI and the $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It split the household into a joint return and a single return for the dependent and added the two taxes. The disabled 27-year-old is a qualifying child claimed on the joint return, producing a $500 credit for other dependents, and has no separate liability in this output."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the dependent's wages in household AGI (about $157,598) and applied personal exemptions that do not exist under the permanent TCJA/OBBBA rules. The correct joint AGI is $112,408 and the correct taxable income is $80,208."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly noted that the dependent's wages are not on the parents' return, then used an aggregation that added the $45,000 anyway. That gave $157,408 of AGI and 22%-bracket tax instead of $112,408 of AGI and $80,208 taxable."
+us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly claimed the $500 credit for other dependents but then added $3,353.74 of tax from a separate return for the same dependent, which double-counts the household unit. It also used a $30,840 standard deduction and a $23,850 bracket edge instead of $32,200 and $24,800."
+us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It treated the disabled 27-year-old as self-supporting and gave them a separate taxed return. A permanently disabled child is a qualifying child at any age and generates the $500 credit for other dependents. It also deducted $3,000 of student-loan interest instead of the $2,500 cap."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed the deductions remove all taxable income, but $112,408 of AGI less the $32,200 standard deduction leaves $80,208 taxable. That produces $9,128.99 of tax before the $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It gave a vague estimate of $6,347 that mentions itemized medical effects, which do not apply because $2,102 of medical costs is below 7.5% of AGI. The correct calculation gives $80,208 taxable, $9,128.99 of tax, and $8,628.99 after the $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It put the dependent's $45,000 of wages on the joint return and also subtracted three sets of $8,389 ESI premiums from wages, ending at $132,241 of AGI. The correct AGI uses only the head's and spouse's box-1 wages less traditional 401(k) deferrals: $112,408."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It reached $124,708 of taxable income by including the dependent's wages in the joint return's income. It also said no nonrefundable credit applies, missing the $500 credit for other dependents for the disabled adult qualifying child."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed all three people's wages on the joint return, giving about $125,398 of taxable income at 22%, and omitted the $190 IRA deduction. Only the head's and spouse's income belongs in the tax unit's AGI ($112,408)."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It applied the $500 credit for other dependents correctly, but its taxable income of $85,208 is $5,000 above the correct $80,208 ($112,408 AGI less the $32,200 standard deduction). That overstated tax by $600."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent's wages on the joint return and subtracted all three people's ESI premiums from wages, producing $132,241 of AGI. The correct AGI is $112,408, from the head's and spouse's wages less 401(k) deferrals, IRA contributions and the $2,500 student-loan cap."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It computed the joint return's $9,128.96 of tax correctly. It then skipped the $500 credit for other dependents for the disabled adult qualifying child and added $3,220 of tax from a separate return the claimed dependent does not file."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted $16,778 of ESI premiums from wages that are already box-1 taxable amounts. That understated AGI at $95,630 instead of $112,408 and taxable income at $63,430 instead of $80,208."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the household's combined wages, including the dependent's $45,000, on the joint return, which pushed taxable income to about $125,398 in the 22% bracket. The tax unit's AGI includes only the head's and spouse's income."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed household-wide wages, including the dependent's, which overstated taxable income well above the correct $80,208. It also gave no clear application of the $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied post-TCJA-sunset rates and exemptions, but OBBBA made the TCJA brackets permanent and set the 2026 joint standard deduction at $32,200. It also taxed the claimed dependent on a separate single return and omitted the $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It used a sunset regime with a 15% bracket, a $16,600 standard deduction and $5,300 exemptions instead of the permanent TCJA/OBBBA rates and the $32,200 deduction. It also added a separate return for the dependent instead of claiming them for the $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It invented a $173 overtime-premium deduction even though nobody works more than 40 hours. It also taxed the disabled 27-year-old on a separate $45,000 single return instead of claiming them as a qualifying child for the $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It used post-TCJA-sunset brackets and a $30,750 standard deduction instead of the OBBBA 2026 values of $32,200 and 10%/12% brackets. It also added a separate $3,820 tax for the dependent, who is claimed on the joint return."
+us,scenario_093,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It got the joint return's $9,129 of tax right. It then added $3,220 from a separate return for the dependent and never applied the $500 credit for other dependents for the permanently disabled adult child."
+us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It submitted no value or explanation for federal_income_tax_before_refundable_credits.
+us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It treated all three earners as one joint return and added the dependent's $45,000 of wages, giving $161,656 of AGI. It also wrongly disallowed the $190 of traditional IRA deductions as phased out."
+us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no reasoning and reported zero. The joint return has $80,208 of taxable income, producing $9,128.99 of tax and $8,628.99 after the $500 credit for other dependents."
+us,scenario_093,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,household_unit_or_filing_status,False,"It reached the correct $112,408 AGI but used a standard deduction of about $31,000 instead of the 2026 value of $32,200. It also applied no nonrefundable credit, missing the $500 credit for other dependents for the disabled adult qualifying child."
+us,scenario_093,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"It computed about $9,139 of joint tax on $80,208 taxable but omitted the $500 credit for other dependents. It then added about $3,225 of tax from a separate return the claimed dependent does not file."
+us,scenario_093,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to the joint return's gross income, giving $157,408 of AGI and 22%-bracket tax. The tax unit's income includes only the head and spouse, so AGI is $112,408 and taxable income is $80,208."
+us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted $16,778 of ESI premiums from wages, deducted $3,000 of student-loan interest instead of the $2,500 cap, and dropped the IRA deduction. It also used a $30,900 standard deduction, omitted the $500 credit for other dependents, and submitted $7,625.75, which matches none of its own computed figures."
+us,scenario_093,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent's wages in AGI and deducted the full $3,000 of student-loan interest. It also wrongly excluded the $500 credit for other dependents from this metric, then submitted $3,778, which contradicts its own computed liability of $13,491."
us,scenario_093,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_093,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It got all six per-person components right ($3,286, $768.50, $4,030, $942.50, $2,790, $652.50), but added them to $11,469.50 instead of $12,469.50. It then submitted $12,348.76, a figure that matches neither its own sum nor its components."
-us,scenario_093,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its reasoning names the correct method, 7.65% employee FICA on $163,000 of combined wages with no Additional Medicare Tax. It never did the multiplication, which gives $12,469.50, and submitted 0 instead."
-us,scenario_093,payroll_tax,claude-opus-4.7,llm_error,other,False,"It correctly computed $10,106 Social Security plus $2,363.50 Medicare for a total of $12,469.50. It then submitted $12,487.50, so the error was in copying its own total into the answer."
-us,scenario_093,payroll_tax,claude-sonnet-5,llm_error,other,False,"Each per-person Social Security and Medicare figure was correct. However, it added the six amounts to $12,622.90 instead of the correct $12,469.50."
-us,scenario_093,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"It subtracted $8,389 of ESI premiums from each worker's wages as if they were pre-tax Section 125 deductions, giving $137,833 of FICA wages. The facts give only gross wages and never describe a pre-tax salary reduction, so the full $163,000 is FICA wages and the tax is $12,469.50."
-us,scenario_093,payroll_tax,deepseek-v4.1-flash,llm_error,payroll_tax_base,False,"It assumed the $8,389 ESI premium came out of each worker's pay pre-tax and lowered the FICA base to $137,833. No such cafeteria-plan reduction is in the facts. Applying 7.65% to the full $163,000 of gross wages gives $12,469.50."
-us,scenario_093,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"It used the correct base ($163,000) and rate (6.2% + 1.45%) but got the multiplication wrong. 7.65% × $163,000 is $12,469.50, not $12,481."
-us,scenario_093,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It treated the three $8,389 ESI premiums ($25,167 total) as Section 125 exclusions and reduced FICA wages to $137,833. The household facts list only gross wages and no pre-tax premium deduction, so FICA applies to the full $163,000, which gives $12,469.50."
-us,scenario_093,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It set each worker's FICA wages to gross pay minus the $8,389 ESI premium ($44,611, $56,611, $36,611), which assumes a pre-tax payroll deduction that isn't in the facts. Using the full gross wages of $53,000, $65,000 and $45,000 gives $12,469.50."
-us,scenario_093,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"It submitted $12,654 with no derivation, which is $184.50 too high. The correct calculation is 6.2% + 1.45% on $163,000 of gross wages, all under the wage base and with no Additional Medicare Tax, which gives $12,469.50."
-us,scenario_093,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It deducted the $8,389 ESI premium from each worker's wages as a pre-tax health deduction and taxed only $137,833. The facts specify gross wages with no cafeteria-plan reduction, so all $163,000 is subject to FICA, which gives $12,469.50."
-us,scenario_093,payroll_tax,gpt-5.4-mini,llm_error,other,False,"It correctly applied Social Security and Medicare to the full $53,000, $65,000 and $45,000 with no Additional Medicare Tax. But it calculated $12,300.30 instead of 7.65% × $163,000 = $12,469.50."
-us,scenario_093,payroll_tax,gpt-5.4-nano,llm_error,other,False,"Its formula, 0.062 × $163,000 + 0.0145 × $163,000, is correct. It got the arithmetic wrong, though: $10,106 + $2,363.50 = $12,469.50, not $9,024."
-us,scenario_093,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"It treated each $8,389 ESI premium as a pre-tax deduction from FICA wages, cutting the base to $137,833. The facts give gross wages with no pre-tax premium reduction, so 7.65% of the full $163,000 is $12,469.50."
-us,scenario_093,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It calculated FICA on $137,833 by assuming the ESI premiums were pre-tax payroll deductions. No cafeteria-plan salary reduction is listed, so the base is the full $163,000 of gross wages, which gives $12,469.50."
-us,scenario_093,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It subtracted the employer-plan premiums from wages before applying 7.65%, which assumes a Section 125 exclusion that isn't in the facts. FICA applies to the full $163,000 of gross wages, which gives $12,469.50."
-us,scenario_093,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation came back, so no answer could be scored."
-us,scenario_093,payroll_tax,minimax-m3,llm_error,other,False,"It submitted 0 with no reasoning, which means no employee FICA at all on $163,000 of W-2 wages. All three workers owe 6.2% Social Security plus 1.45% Medicare, for a total of $12,469.50."
-us,scenario_093,payroll_tax,qwen-3.7-max,llm_error,other,False,"It correctly computed $4,054.50 + $4,972.50 + $3,442.50 = $12,469.50. It then submitted $12,553.50, so the error was in copying its own total into the answer."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"Its ~$79,800 taxable income is close to $79,776.80, but it estimated the tax by running the graduated schedule once on joint income. Missouri splits taxable income between the spouses ($34,743.94 and $45,032.86) and taxes each share under the 2026 brackets. That approach loses the second spouse's lower-bracket savings and skips the $431.45 federal-tax deduction, giving $3,506 instead of $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It used MO AGI of $156,408, $44,000 above the couple's $112,408. The dependent has $0 MO AGI, so joint AGI stays below $125,000 and the 5% federal-tax deduction ($431.45) still applies. It also ran $124,208 through one schedule instead of taxing each spouse's share separately."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It started from gross wages plus interest ($119,156) without excluding the $4,058 of traditional 401(k) deferrals, the IRA deductions, or student-loan interest. It used a $27,900 standard deduction instead of 2026's $32,200, then applied an invented 4.04% effective rate instead of the 2026 Missouri brackets per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It reported $0 with no computation. MO AGI of $112,408.25 minus the $32,200 standard deduction and the $431.45 federal-tax deduction leaves $79,776.80 taxable, which produces $3,388.25 under the 2026 brackets. The answer ignores that positive taxable income."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It listed the $4,058 of traditional 401(k) deferrals but never subtracted them, so it got AGI of $116,466 instead of $112,408.25. It used a ~$30,000 standard deduction instead of $32,200, left out the $431.45 federal-tax deduction, and applied the schedule once to joint income instead of per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"Its setup was close: $80,408 taxable, using a $31,500 standard deduction instead of $32,200 and the full $3,000 of student-loan interest instead of the $2,500 cap. Its own formula gives about $3,640, but it then raised that to $4,126 with a 'graduated bracket adjustment', even though lower brackets reduce tax. It also dismissed the federal-tax deduction, which is 5% of federal liability ($431.45) at this AGI."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It added a separate Missouri tax of about $1,400 on the dependent's $45,000 of wages. The frozen $3,388.25 covers the couple's return. The adjudication requires a separate return for the dependent and excludes this output because household scope is unresolved."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It used MO AGI of $157,408, adding the dependent's $45,000 of wages to the joint return. That pushed AGI over $125,000, which eliminated the federal-tax deduction and raised taxable income to $125,208 instead of $79,776.80."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the $16,778 of employer-sponsored insurance premiums from wages, giving AGI of $95,130 instead of $112,408.25. It also subtracted a $4,400 personal exemption that Missouri does not allow and deducted the whole $7,339 of federal tax instead of the AGI-tiered percentage. It then applied a flat 4.7% instead of the graduated brackets per spouse."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It put MO taxable income at about $102,000, roughly $22,000 above the couple's taxable income in the engine. The correct figure is $79,776.80, after the $32,200 standard deduction and the $431.45 federal-tax deduction."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It got MO AGI ($112,408) and the $32,200 standard deduction right but left out the federal-tax deduction: $431.45, which is 5% of federal liability for AGI of at least $100,000 and below $125,000. It also applied the 2025 schedule once to $80,208 of joint income instead of taxing each spouse's share under the 2026 brackets."
-us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It used AGI of $160,098, which includes the dependent's wages, and subtracted $6,300 of personal exemptions that Missouri does not allow. It used a $30,600 standard deduction instead of $32,200 and a flat 4.5% rate instead of the graduated 2026 schedule, which tops out at 4.7%."
-us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It added a separate single-filer tax of $1,056.06 for the dependent, who has $0 MO AGI inside the couple's tax unit. It also used 2024 brackets and standard deductions instead of the $32,200 standard deduction and the 2026 brackets."
-us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It added the couple's taxable income to a separate dependent return, for $99,241 in total versus $79,776.80 for the couple alone. It also subtracted exemptions Missouri does not allow and used a 4.6% top rate instead of the 2026 schedule's 4.7%."
-us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the $8,389 employer-sponsored insurance premium from each worker's wages, giving couple AGI of $95,630 instead of $112,408.25. It also added $801.81 for the dependent's separate return. The adjudication requires that return and leaves its inclusion in the requested output unresolved, so the output is excluded from scoring."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It treated the federal-tax deduction as full federal liability up to the $10,000 or $5,000 cap, instead of the AGI-tiered percentage (5%, or $431.45, here). It also computed a separate single return for the dependent and used a 4.8% rate instead of the 2026 top rate of 4.7%."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. $6,055 implies roughly $129,000 taxed at 4.7%, which matches adding the dependent's $45,000 of wages to the couple's income. The couple's taxable income is $79,776.80, and the tax is $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It guessed a round $4,000 from a ~4.5% rate on 'combined' household income. The correct path is $79,776.80 of couple taxable income (after the $32,200 standard deduction and $431.45 federal-tax deduction), with each spouse's share taxed under the 2026 brackets."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It refers to 'standard deductions' in the plural, and its $4,166 total is $778 above the couple's $3,388.25. Its reasoning does not itemize a separate Missouri liability for the dependent. The adjudication leaves inclusion of the dependent's required separate return unresolved."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. $5,321 is about $1,930 above the frozen $3,388.25 for the couple's return. The adjudication leaves inclusion of the dependent's required return unresolved, so this output is excluded from scoring."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"Its $3,421 is about $33 above the frozen value for the couple's return. The engine's couple-return calculation uses $112,408.25 − $32,200 − $431.45 = $79,776.80, split $34,743.94 / $45,032.86 between the spouses and taxed under the 2026 brackets for $1,452.33 + $1,935.91. Its reasoning does not state its bracket thresholds or federal-tax deduction amount. The output is excluded from scoring because household scope is unresolved."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"Its $4,850 total for 'household tax units' is about $1,460 above the frozen $3,388.25 for the couple's return; its reasoning does not itemize a separate dependent liability. The adjudication requires a separate federal and state return for the dependent and excludes this output because household scope is unresolved."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. $5,250 is about $1,860 above the frozen $3,388.25 for the couple's return. The adjudication leaves inclusion of the dependent's required return unresolved, so this output is excluded from scoring."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,household_unit_or_filing_status,False,"It used AGI of $157,408, which includes the dependent's wages, and took the full $10,000 federal-tax deduction cap. At the correct $112,408.25 AGI the deduction is 5% of federal liability ($431.45). It also subtracted a $1,200 dependent deduction that does not exist in the 2026 Missouri calculation."
-us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It added a separate single return for the dependent ($30,400 taxable) and used the 2024 standard deductions ($29,200 / $14,600) instead of 2026's $32,200. It skipped the IRA and federal-tax deductions and applied a flat 4.7% with no graduated lower brackets."
-us,scenario_093,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It added $1,356 of single-filer tax for a dependent who has $0 MO AGI in the couple's unit. It taxed the couple's $79,708 at a flat 4.7% instead of applying the 2026 graduated brackets to each spouse's share. It also deducted the full $3,000 of student-loan interest instead of the $2,500 cap and left out the $431.45 federal-tax deduction."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed the standard deduction wipes out all income. Against $112,408.25 of MO AGI, the $32,200 standard deduction and $431.45 federal-tax deduction leave $79,776.80 taxable, which produces $3,388.25 of tax."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It gave a rough approximation ($3,427) without computing the $431.45 federal-tax deduction. It also did not apply the 2026 brackets separately to each spouse's share ($34,743.94 and $45,032.86) of $79,776.80 taxable income."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It used AGI of $157,598, which includes the dependent's wages, and subtracted $5,400 of personal and dependent exemptions that Missouri does not allow. It applied a 4.5% top rate instead of the 2026 schedule's 4.7%."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"Its $124,708 of taxable income equals $156,908 − $32,200, meaning the couple's income plus the dependent's $45,000 of wages. With the dependent at $0 MO AGI, taxable income is $79,776.80 and the $431.45 federal-tax deduction applies."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It said income was above the range for the federal-tax deduction; the engine's zero deduction rate begins at AGI of $125,000. The couple's $112,408.25 MO AGI is in the 5% band, giving a $431.45 deduction and $79,776.80 taxable."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted Missouri personal exemptions that do not exist in 2026 and over-sized the federal-tax deduction, which is only 5% of federal liability ($431.45) at this AGI. This took about $4,200 too much off taxable income compared with $79,776.80."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"Its $99,041 of taxable income implies MO AGI of about $132,000 instead of $112,408.25, which led it to conclude wrongly that no federal-tax deduction applies. It also subtracted a $1,000 health-insurance deduction that is not part of this calculation."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It counted only the spouses, one of the scope readings the adjudication leaves unresolved; this output is excluded from scoring. For the couple's return, the engine deducts 5% of federal liability ($431.45), and its $3,124 implies taxable income about $5,600 below $79,776.80. The engine's 5% deduction band covers AGI of at least $100,000 and below $125,000."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It added tax on a separate return for the dependent with about $28,900 taxable. On the couple's side it used $80,208 taxable, leaving out the $431.45 federal-tax deduction."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"Its figures add up to $95,630 of AGI ($61,776.10 + $1,653.90 + $32,200), which shows it subtracted the $16,778 of employer-sponsored insurance premiums from wages. MO AGI is $112,408.25, and at that AGI the federal-tax deduction is 5% ($431.45), not $1,653.90."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It said no federal-tax deduction applies 'at this income'; the engine applies a zero deduction rate at AGI of at least $125,000. The couple's $112,408.25 MO AGI gets a $431.45 deduction and $79,776.80 taxable."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"$3,780 equals a flat 4.7% on about $80,400. It skipped the graduated lower brackets for each spouse and the $431.45 federal-tax deduction, which together bring the tax on $79,776.80 down to $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It computed a separate return for the dependent ($36,478 taxable, $1,547 tax) and overstated the couple's taxable income at $95,365 versus $79,776.80. It also used a 4.8% top rate instead of 4.7%."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It added a $1,555 single return for the dependent and overstated the couple's taxable income at $95,808. The correct figure is $79,776.80, after the 401(k), IRA and student-loan reductions, the $32,200 standard deduction, and the $431.45 federal-tax deduction."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It added $1,196 of tax for the dependent on a separate return. On the couple's $80,035 it applied the schedule once instead of per spouse and dropped most of the $431.45 federal-tax deduction."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It computed a separate $1,354 tax for the dependent, who has $0 MO AGI within the couple's unit. It also used the outdated 4.95% top rate instead of the 2026 schedule's 4.7%."
-us,scenario_093,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It added $1,047 of single-unit tax for the dependent, who has $0 MO AGI within the couple's tax unit. Its joint-unit tax of $3,227 is also too low because it deducted more federal tax than the 5% ($431.45) Missouri allows at this AGI."
-us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,prompt_ambiguity,household_unit_or_filing_status,False,"It used outdated Missouri parameters: a 5.3% top rate, $6,000 of personal exemptions, and a 35% federal-tax deduction. In this joint-return calculation the top rate is 4.7%, there are no personal or dependent exemptions, and the federal-tax deduction is 5% ($431.45) for AGI of at least $100,000 and below $125,000. It then raised its result with an invented 'indexation' adjustment."
-us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It started from $161,656 of household AGI, equal to household wages and taxable interest of $164,156 less $2,500. That eliminated the federal-tax deduction and raised taxable income to $129,456, against the couple's $79,776.80."
-us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It reported $0 with no derivation. MO AGI of $112,408.25 minus $32,200 and $431.45 leaves $79,776.80 taxable, which produces $3,388.25."
-us,scenario_093,state_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,household_unit_or_filing_status,False,"It reached $3,390 through two errors that cancel out. It deducted about $8,000 of federal tax, treating the $10,000 cap as the limit instead of the 5% rule that gives $431.45, which left $72,100 taxable. It then applied a flat 4.7% instead of the graduated 2026 brackets per spouse on $79,776.80."
-us,scenario_093,state_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"It added $1,251 of tax for the dependent on a separate return. On the couple's side it used $80,208 taxable without the $431.45 federal-tax deduction."
-us,scenario_093,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent's wages in AGI ($157,408) and subtracted $3,600 of $1,200 exemptions that Missouri does not allow. That raised taxable income to $121,608, against $79,776.80."
-us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent's $45,000 of wages to joint AGI, skipped the 401(k), IRA and student-loan reductions, and used a $26,600 standard deduction and a 4.95% rate. It then reported $11,145.20, which does not follow from its own $6,809 figure."
-us,scenario_093,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It started from about $124,942 of taxable income, which includes the dependent's income, and applied a 4.5% rate. The couple's MO taxable income is $79,776.80, taxed per spouse under 2026 brackets that top out at 4.7%."
+us,scenario_093,payroll_tax,claude-fable-5.2,llm_error,payroll_tax_base,False,"Took traditional 401(k) elective deferrals out of FICA wages ($49,027 + $64,915 + $45,000 = $158,942), but under IRC §3121(v)(1)(A) elective deferrals stay in Social Security and Medicare wages. With full gross wages of $163,000 at 7.65%, the tax is $12,469.50, not $12,159.06."
+us,scenario_093,payroll_tax,claude-haiku-4.5,llm_error,other,False,"Its per-person amounts were correct ($3,286 + $768.50 + $4,030 + $942.50 + $2,790 + $652.50), but it added them up to $11,469.50 instead of $12,469.50. It then submitted a third, unrelated figure of $12,348.76."
+us,scenario_093,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its reasoning applies 7.65% to $163,000 of combined wages, which gives $12,469.50, but it submitted 0. The answer contradicts its own method."
+us,scenario_093,payroll_tax,claude-opus-4.7,llm_error,other,False,"Its reasoning computed the correct total of $10,106 Social Security plus $2,363.50 Medicare = $12,469.50, but it submitted $12,487.50. This is a transcription error in the final value."
+us,scenario_093,payroll_tax,claude-sonnet-5,llm_error,other,False,"Its six per-person Social Security and Medicare amounts were correct, but it summed them to $12,622.90 instead of $12,469.50. This is a pure addition error."
+us,scenario_093,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"Subtracted the $8,389 employer-sponsored insurance premium from each worker's FICA wages as if it were a pre-tax salary reduction, but the prompt lists no cafeteria-plan reduction. The FICA base is the full $163,000 of gross wages, so the tax is $12,469.50, not 7.65% of $137,833."
+us,scenario_093,payroll_tax,deepseek-v4.1-flash,llm_error,payroll_tax_base,False,"Reduced each worker's wages by the $8,389 employer-sponsored insurance premium before applying FICA, which treats the premium as a Section 125 salary reduction. The prompt gives no such fact, and the payroll tax base is the full $163,000, which gives $12,469.50."
+us,scenario_093,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"Stated the correct method, 7.65% of $163,000, but reported $12,481 instead of the correct product of $12,469.50. This is an arithmetic error."
+us,scenario_093,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Treated the $25,167 in combined employer-sponsored insurance premiums as Section 125 pre-tax deductions and cut FICA wages to $137,833. The prompt does not describe the premiums as salary reductions. FICA applies to the full $163,000 of gross wages, which gives $12,469.50."
+us,scenario_093,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"Computed FICA on wages after subtracting the $8,389 employer-sponsored insurance premium for each person ($44,611, $56,611, $36,611), but the premiums are not salary reductions in the prompt. FICA applies to gross wages of $53,000, $65,000 and $45,000, which total $12,469.50."
+us,scenario_093,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"Gave $12,654 with no derivation. The correct calculation is 7.65% of $163,000 of gross wages, which is $12,469.50, so $12,654 overstates it by $184.50 and comes from an arithmetic error, not a different tax base."
+us,scenario_093,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"Deducted the $8,389 employer-sponsored insurance premium from each worker's FICA wages, which brings the base to $137,833. Nothing in the prompt makes these premiums pre-tax salary reductions, so the base is the full $163,000 and the tax is $12,469.50."
+us,scenario_093,payroll_tax,gpt-5.4-mini,llm_error,other,False,"Correctly said Social Security and Medicare apply to the full $53,000, $65,000 and $45,000 with no Additional Medicare Tax, but reported $12,300.30. The correct product is 7.65% × $163,000 = $12,469.50, so this is a miscalculation."
+us,scenario_093,payroll_tax,gpt-5.4-nano,llm_error,other,False,"Wrote the correct formula, 0.062 × $163,000 + 0.0145 × $163,000, but evaluated it as $9,024 instead of $10,106 + $2,363.50 = $12,469.50. This is an arithmetic error."
+us,scenario_093,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"Treated the employer-sponsored insurance premiums as pre-tax salary reductions and used $137,833 as FICA wages. The prompt lists only gross wages with no cafeteria-plan election, so FICA applies to the full $163,000 and gives $12,469.50."
+us,scenario_093,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"Took $8,389 per worker in employer-sponsored insurance premiums out of FICA wages, which leaves $137,833. The payroll tax base is full gross wages of $163,000, which gives 7.65% = $12,469.50."
+us,scenario_093,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"Subtracted employer-plan premiums from wages before applying FICA, which gives $137,833. The premiums are not salary reductions under the prompt's facts, so employee Social Security and Medicare apply to the full $163,000, for $12,469.50."
+us,scenario_093,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"Submitted no value and no explanation for payroll_tax. The expected value was 7.65% of $163,000 of gross wages, which is $12,469.50."
+us,scenario_093,payroll_tax,minimax-m3,llm_error,other,False,"Submitted 0 with no reasoning, which ignores employee FICA completely. All three adults have wage income, so 6.2% Social Security plus 1.45% Medicare on $163,000 gives $12,469.50."
+us,scenario_093,payroll_tax,qwen-3.7-max,llm_error,other,False,"Correctly computed $4,054.50 + $4,972.50 + $3,442.50 = $12,469.50 in its reasoning, but submitted $12,553.50. This is a transcription error in the final value."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model reached about the right taxable base (~$79,800) but left out Missouri's federal income tax deduction ($431.45, which is 5% of federal liability at MO AGI of $100k–$125k). It also ran the combined income through the brackets once, when Missouri splits the $79,776.80 between the spouses by income share and taxes each share separately. It also took $3,000 of student loan interest instead of the $2,500 cap."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model added the dependent's $45,000 wages to the joint MO AGI ($156,408). The dependent's MO AGI is $0, and the spouses' MO AGI is $112,408.25. The inflated AGI went above $125k, which removed the 5% federal-tax deduction ($431.45), and the model taxed $124,208 on one schedule instead of $79,776.80 split between the spouses."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"The model built a $156,408 MO AGI that includes the dependent's $45,000 wages and split the tax across three people. Only the head and spouse have Missouri AGI ($112,408.25). That AGI keeps the 5% federal-tax deduction ($431.45) and gives $79,776.80 of taxable income split between the two spouses."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model started from gross wages plus interest ($119,156). It never subtracted the $4,058 of traditional 401(k) deferrals, the $190 IRA deduction or the $2,500 student loan interest deduction, and it used a $27,900 standard deduction instead of the 2026 MFJ amount of $32,200. It then applied an invented 4.04% effective rate instead of Missouri's graduated brackets to each spouse's share of $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model reported zero tax. The spouses' MO AGI of $112,408.25, minus the $32,200 standard deduction and the $431.45 federal-tax deduction, leaves $79,776.80 of taxable income. Taxed per spouse on the graduated schedule, that comes to $3,388.25, and no Missouri nonrefundable credit offsets it."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model noted the $4,058 of traditional 401(k) deferrals but never subtracted them, which produced an AGI of $116,466 instead of $112,408.25. It also used a $30,000 standard deduction instead of $32,200, left out the $431.45 federal-tax deduction, and taxed the combined income on one schedule instead of each spouse's share separately."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"The model's taxable income of $80,408 used a $31,500 standard deduction instead of $32,200 and $3,000 of student loan interest instead of the $2,500 cap. Its own bracket math gives about $3,640, which it then raised to $4,126 with no basis. It also left out the $431.45 federal-tax deduction and the per-spouse split that produce $3,388.25."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model estimated the joint tax at about $3,400, then added a separate Missouri tax of about $780 on the dependent's $45,000 wages. The dependent is in the couple's tax unit with $0 Missouri AGI and owes nothing, so the total is just the spouses' $1,452.33 + $1,935.91."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model's MO AGI of $157,408 adds the dependent's $45,000 wages to the spouses' $112,408.25, but the dependent's MO AGI is $0. The inflated AGI removed the 5% federal-tax deduction ($431.45) and raised taxable income to $125,208 instead of $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted each spouse's $8,389 employer insurance premium from wages as a pre-tax reduction, which the wage inputs do not support, and added a $4,400 Missouri personal exemption that Missouri no longer allows. It deducted the full $7,339 of federal tax instead of 5% of federal liability ($431.45) and applied a flat 4.7%. Its taxable base ended up far below $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"The model put MO taxable income at about $102,000. The spouses' MO AGI of $112,408.25, minus the $32,200 standard deduction (not its $30,000 estimate) and the $431.45 federal-tax deduction, leaves $79,776.80. That amount is then split between the spouses and each share is taxed separately."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model correctly reached MO AGI of $112,408 and the $32,200 standard deduction. It skipped Missouri's federal income tax deduction (5% of the $8,629 federal liability, $431.45) and taxed $80,208 once on the schedule. Missouri's combined return taxes each spouse's share ($34,743.94 and $45,032.86) separately, which uses the low brackets twice and cuts the tax by about $181."
+us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $160,098 AGI includes the dependent's $45,000 wages, but the dependent's MO AGI is $0. It also subtracted three $2,100 personal exemptions, which Missouri has repealed, and used a flat 4.5% instead of graduated brackets that top out at 4.7%. Its taxable income was $123,198 instead of $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate single-filer tax of $1,056.06 for the dependent, who is in the couple's tax unit with $0 MO AGI. For the joint return ($3,221.32) it used 2024 brackets and standard deductions instead of the 2026 $32,200 deduction, the 5% federal-tax deduction and the per-spouse 2026 brackets."
+us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $99,241 of total taxable income includes the dependent's wages. Only the spouses' $79,776.80 is taxable, because the dependent's MO AGI is $0. It also used a 4.6% top rate instead of 4.7%."
+us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model cut the couple's AGI to $95,630 by subtracting $16,778 of employer insurance premiums from wages, but the spouses' MO AGI is $112,408.25 with no such exclusion. It then added a separate $801.81 tax for the dependent, who is in the couple's unit with $0 MO AGI."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model ran a separate Single return for the dependent, who is in the couple's tax unit with $0 MO AGI. It deducted federal tax up to the $10,000 cap, but at MO AGI of $112,408.25 only 5% of federal liability ($431.45) is deductible. It also used a 4.8% top rate instead of 4.7%."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no derivation. $6,055 is about $2,700 above the correct figure, which fits taxing the dependent's $45,000 wages along with the spouses' income. Only the spouses' $79,776.80 of taxable income (after the $32,200 standard deduction and the $431.45 federal-tax deduction) is taxed, split between the spouses."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"The model applied a rough 4.5% rate to combined household income that includes the dependent's wages. The dependent's MO AGI is $0, and Missouri taxes each spouse's share of the $79,776.80 on graduated brackets that top out at 4.7%."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $4,166 is about $778 above the correct figure, which matches a separate Missouri tax on the dependent's wages. The dependent belongs to the couple's tax unit with $0 MO AGI, so only the spouses' $79,776.80 is taxed, with each spouse's share taxed separately."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no derivation. $5,321 fits putting the dependent's $45,000 wages into the taxed base. The dependent's MO AGI is $0, and only the spouses' $79,776.80 of taxable income is taxed, which gives $3,388.25."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model used the right structure (standard deduction plus federal-tax deduction) but came out $32.75 high, which is about $700 of extra taxable income at 4.7%. The correct base is $112,408.25 − $32,200 − $431.45 = $79,776.80, split $34,743.94/$45,032.86 between the spouses and taxed per spouse on the 2026 brackets (top 4.7% bracket starting near $9,436)."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed several 'household tax units', so it added the dependent's wages as a separate taxable unit, which put it about $1,460 too high. The dependent is in the couple's unit with $0 MO AGI, and only the spouses' $79,776.80 of taxable income is taxed."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no derivation. $5,250 is about $1,860 too high, which fits taxing the dependent's $45,000 wages. The dependent's MO AGI is $0, and the spouses' $79,776.80 of taxable income, taxed per spouse, gives $3,388.25."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,household_unit_or_filing_status,False,"The model started from $157,408, which includes the dependent's $45,000 wages, even though the dependent's MO AGI is $0. It took a $10,000 federal-tax deduction (0% is allowed above $125k AGI, and 5% at the spouses' actual $112,408.25) and a $1,200 dependent deduction that Missouri has repealed."
+us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed the dependent as a separate single filer, although the dependent is in the couple's unit with $0 MO AGI. It used 2024 standard deductions instead of the 2026 $32,200, left out the $431.45 federal-tax deduction, and applied a flat 4.7% instead of graduated brackets to each spouse's share."
+us,scenario_093,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate $1,356 tax on the dependent, who is in the couple's unit with $0 MO AGI. It also applied a flat 4.7% to $79,708 instead of graduated brackets to each spouse's share of $79,776.80 (which already reflects the $431.45 federal-tax deduction)."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"The model claimed the standard deduction wipes out all taxable income. The $32,200 MFJ deduction plus the $431.45 federal-tax deduction leave $79,776.80 of the spouses' $112,408.25 MO AGI taxable, which produces $3,388.25."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no derivation and came out $38.75 high, which is about $824 of extra taxable income at 4.7%. The correct base is MO AGI of $112,408.25 minus the $32,200 standard deduction and the $431.45 federal-tax deduction, split between the spouses and taxed per spouse on the 2026 brackets."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $157,598 AGI includes the dependent's wages, but the dependent's MO AGI is $0. It also subtracted $5,400 of personal/dependent exemptions that Missouri has repealed and used a 4.5% top rate instead of 4.7%."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model's taxable income of $124,708 comes from counting the dependent's $45,000 wages along with the spouses' income. The dependent's MO AGI is $0, and taxable income is $79,776.80 after the standard deduction and the 5% federal-tax deduction."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model said income is above the federal-tax deduction range, which requires MO AGI over $125k. It only gets there by counting the dependent's wages. The spouses' MO AGI of $112,408.25 is in the 5% band ($431.45 deduction), and each spouse's share is taxed separately."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"The model subtracted personal exemptions, which Missouri repealed starting in 2018. That understated taxable income by about $4,200, so the result is $195.81 too low. Taxable income is $112,408.25 − $32,200 − $431.45 = $79,776.80, with no exemption."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"The model said no federal-tax deduction applies at this AGI, which treats MO AGI as above $125k by counting income beyond the spouses' $112,408.25 (the dependent's wages). Its $99,041 of taxable income is about $19,300 above the correct $79,776.80, which keeps the $431.45 federal-tax deduction."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"The model's $3,124 is $264.25 too low, which is about $5,600 of excess deductions. At the spouses' MO AGI of $112,408.25, the federal-tax deduction is capped at 5% of the $8,629 federal liability ($431.45), which leaves $79,776.80 taxable and taxed per spouse."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model taxed the dependent's $28,900 separately, although the dependent is in the couple's unit with $0 MO AGI. For the couple it used $80,208 with no federal-tax deduction, when the 5% deduction ($431.45) lowers taxable income to $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"The model's AGI of $95,630 subtracts both spouses' $8,389 employer insurance premiums from wages, but MO AGI is $112,408.25 with no such exclusion. The lower AGI also moved it into the 15% federal-tax band ($1,653.90) instead of the 5% band ($431.45)."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,household_unit_or_filing_status,False,"The model concluded there was no federal-tax deduction, which means it pushed AGI above $125k by including the dependent's wages. The spouses' MO AGI of $112,408.25 qualifies for the 5% deduction ($431.45), giving $79,776.80 of taxable income, which is taxed per spouse."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model gave no derivation. $3,780 matches taxing about $84,300 on a single bracket schedule. The correct base is $79,776.80 ($112,408.25 − $32,200 − $431.45), split by income share and taxed separately for each spouse."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate $1,547 tax for the dependent, who is in the couple's unit with $0 MO AGI. For the couple it left $95,365 taxable by subtracting only about a single-filer standard deduction instead of the $32,200 MFJ amount, and it used a 4.8% top rate instead of 4.7%."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model's MFJ taxable income of $95,808 is $111,908 minus the single standard deduction ($16,100) instead of the $32,200 MFJ amount. It also added a separate $1,555 tax on the dependent, who has $0 MO AGI inside the couple's tax unit."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate $1,196 single-filer tax for the dependent, who is in the couple's unit with $0 MO AGI. For the couple it left out the $431.45 federal-tax deduction and taxed $80,035 once instead of taxing each spouse's share of $79,776.80 on the 2026 brackets."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"The model computed a separate $1,354 tax for the dependent, who is in the couple's unit with $0 MO AGI. It used an outdated 4.95% top rate instead of the 2026 rate of 4.7%, and it left out the 5% federal-tax deduction and the per-spouse split."
+us,scenario_093,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"The model treated the dependent as a separate single filing unit owing about $1,047, but the dependent is in the couple's tax unit with $0 MO AGI. Only the spouses' tax ($1,452.33 + $1,935.91) counts."
+us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,prompt_ambiguity,household_unit_or_filing_status,False,"The model made up Missouri parameters: a $13,000 standard deduction, three $2,000 personal exemptions, a 35% federal-tax deduction, and brackets with 5.3–5.5% rates. The actual rules are the $32,200 MFJ standard deduction, no exemptions, a 5% federal-tax deduction ($431.45) at $112,408.25 AGI, and a top rate of 4.7% applied to each spouse's share."
+us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"The model's household AGI of $161,656 includes the dependent's $45,000 wages (and leaves the 401(k) deferrals in), even though the dependent's MO AGI is $0. That AGI removed the 5% federal-tax deduction and inflated taxable income to $129,456 instead of $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"The model reported $0 with no reasoning. The spouses' $79,776.80 of MO taxable income (after the $32,200 standard deduction and the $431.45 federal-tax deduction) is taxed at graduated rates up to 4.7% for $3,388.25."
+us,scenario_093,state_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,household_unit_or_filing_status,False,"The model deducted roughly $8,100 of federal tax (taxable income about $72,100), but at MO AGI of $112,408.25 only 5% of federal liability ($431.45) is deductible. It also applied a flat 4.7% instead of graduated brackets to each spouse's share. The two errors offset each other, leaving it $1.75 off."
+us,scenario_093,state_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"The model added a separate $1,251 tax on the dependent, who is in the couple's unit with $0 MO AGI. For the couple it taxed $80,208 once, leaving out the $431.45 federal-tax deduction and the per-spouse allocation that brings the joint tax to $3,388.25."
+us,scenario_093,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"The model started from $157,408 of AGI that includes the dependent's wages, but the dependent's MO AGI is $0. It also subtracted three $1,200 exemptions that Missouri has repealed, giving $121,608 of taxable income instead of $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model put all three people's wages into one $164,156 AGI, skipped the 401(k), IRA and student loan subtractions, and used a $26,600 standard deduction and a 4.95% rate. Its final $11,145.20 doesn't even match its own $6,809 estimate. Only the spouses' $79,776.80 is taxable, at rates up to 4.7%."
+us,scenario_093,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"The model's taxable income of about $124,942 includes the dependent's wages, but the dependent's MO AGI is $0 and the spouses' taxable income is $79,776.80. It also used a 4.5% top rate instead of 4.7%."
us,scenario_095,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the full $14,022 of Social Security in AGI and then applied an age-65 single-filer standard deduction. Social Security is not taxable at this provisional-income level, and the surviving-spouse filing-status deduction eliminates the resulting $15,286.89 AGI."
us,scenario_095,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model correctly reached approximately $15,286 of AGI but treated the head as a single filer and applied a $9,800 deduction. The surviving-spouse filing status provides the larger applicable standard deduction, reducing taxable income to zero."
us,scenario_095,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model correctly concluded that none of the Social Security is taxable, but then retained it in its $29,308 AGI and computed taxable income from that inflated figure. Excluding the nontaxable Social Security produces $15,286.89 of AGI, which the surviving-spouse standard deduction fully offsets."
@@ -8408,10 +8498,11 @@ us,scenario_095,state_income_tax_before_refundable_credits,minimax-m3,llm_error,
us,scenario_095,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"Taxed $2,198 at 1.4% after exemptions. It ignored that NJ income tax is zero whenever NJ gross income, here $4,198, is at or below the $10,000 ($20,000 surviving-spouse) filing threshold."
us,scenario_095,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Correctly excluded Social Security and the pension distribution, then taxed $2,198 at 1.4%. It omitted NJ's low-income exemption, under which $4,198 of NJ gross income carries no tax."
us,scenario_095,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Capped the retirement exclusion at an outdated $10,000 instead of fully excluding the $11,385 under the current exclusion, then taxed $3,394.66 at 1.4%. It also missed the gross-income threshold: total NJ income of $15,583 is under the $20,000 surviving-spouse threshold, and the post-exclusion $4,198 is under $10,000, so tax is zero."
-us,scenario_095,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"Counted the NJ ANCHOR homeowner benefit as a $1,500 refundable income tax credit. ANCHOR is a separate property-tax relief payment, not part of the NJ-1040 refundable credits (NJ EITC, NJ Child Tax Credit, property tax credit). A mortgage balance is not property tax paid, and unlisted property taxes are $0. The model also skipped the NJ EITC, which phases out to $0 at this household's AGI of about $15,286."
-us,scenario_095,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"Correctly found the age-65+ childless NJ EITC pathway and the $298.45 phase-in amount (7.65% × $3,901 net SE earnings), but then took 40% of that without applying the phase-out. The phase-out uses the greater of AGI or earned income, and AGI is about $15,286 because it includes the $11,385 401(k) distribution. With the non-joint threshold of about $11k, the reduction of 7.65% × (AGI − threshold) exceeds $298.45, so the hypothetical federal credit and the NJ EITC are both $0."
-us,scenario_095,state_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Applied 40% to the phase-in amount alone ($298.45 = 7.65% × $3,901 of earnings after the deductible half of SE tax) and never ran the phase-out. The phase-out runs on AGI of about $15,286, which includes the $11,385 401(k) distribution. Because that distribution alone is above the roughly $11k non-joint threshold, the reduction exceeds the phase-in amount and the hypothetical federal credit, and therefore the NJ EITC, is $0."
-us,scenario_095,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no value or explanation for state_refundable_credits, so there was nothing to score. The correct answer is $0: the age-65+ childless NJ EITC is fully phased out at AGI of about $15,286, and with no property tax or rent there is no NJ property tax credit."
+us,scenario_095,state_refundable_credits,claude-fable-5.2,llm_error,age_disability,False,"The model assumed the NJ EITC has no upper age limit for childless filers and applied 40% to a recomputed childless credit of 7.65% × $3,901.42 = $298.46. Under PolicyEngine's rules, the NJ EITC is 40% of the federal EITC. The federal EITC is $0 because a 73-year-old childless filer is over the age-64 limit, so the NJ credit and state_refundable_credits are both $0."
+us,scenario_095,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model counted the NJ ANCHOR property-tax relief benefit as a $1,500 refundable income-tax credit. ANCHOR is a separate property-tax rebate program, not part of nj_refundable_credits, and no property taxes were listed anyway. The only possible refundable NJ credit, the NJ EITC, is $0 because the federal EITC it is based on is $0 for a childless filer over 64."
+us,scenario_095,state_refundable_credits,gpt-6-astra,llm_error,age_disability,False,"The model claimed New Jersey allows the childless EITC for seniors and took 40% of an age-adjusted federal credit to get $119.38. PolicyEngine makes no age adjustment for a 73-year-old childless filer: the NJ EITC is 40% of the actual federal EITC, which is $0 because of the federal age-64 cap. That makes the state refundable credit $0."
+us,scenario_095,state_refundable_credits,gpt-6.1-sol,llm_error,age_disability,False,"The model claimed New Jersey extends its childless EITC to filers aged 65 and over and computed 40% of a hypothetical credit on earnings net of half the SE tax. Under PolicyEngine's rules, the NJ EITC is 40% of the actual federal EITC. The federal EITC is $0 for a childless filer aged 73, so NJ refundable credits total $0."
+us,scenario_095,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for state_refundable_credits, so there was nothing to score. The correct derivation gives $0: the NJ EITC is 40% of a federal EITC that is $0 for a childless 73-year-old, and no other NJ refundable credit applies."
us,scenario_095,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_098,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated all $30,595 of Social Security as taxable when it set AGI to $35,035. After the farm rental loss offsets most of the pension, provisional income is below the Social Security taxation threshold, leaving $4,440 of AGI that the standard deduction fully eliminates."
us,scenario_098,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model counted all Social Security and the $1,100 energy subsidy as taxable income, omitted the $11,100 farm rental loss, and used an inapplicable $13,500 standard deduction. The farm loss leaves $4,440 of non-Social-Security income, Social Security remains nontaxable under the provisional-income test, and the standard deduction reduces taxable income to zero; its claimed $950 age credit does not repair those upstream errors."
@@ -8455,154 +8546,157 @@ us,scenario_099,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligi
us,scenario_099,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the under-five age requirement as the complete WIC eligibility test. Child 2 satisfies the categorical age rule, but household income of approximately $159,676 exceeds the 185%-of-poverty income limit, making the child ineligible."
us,scenario_099,child2_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model inferred eligibility solely from child 2's age and never applied WIC's household-income test. The household's approximately $159,676 annual income is above the applicable 185%-of-poverty limit, so the categorical age condition does not produce eligibility."
us,scenario_099,child2_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output for child2_wic_eligible, violating the required submission contract. It therefore never reported the result produced by applying the WIC income limit: value 0."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It applied the old 20% CDCC rate ($1,200) instead of the OBBBA 32% rate ($1,920). It also deducted the $144 IRA and $1,575 of auto loan interest, which the reference does not allow, and used a $32,600 standard deduction instead of $32,200. Its own work came to $10,965, but it submitted $12,103, which matches nothing it computed."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It got the CTC ($4,400) and CDCC ($1,920) right but left out OBBBA's $2,000 charitable deduction for non-itemizers. That left taxable income about $1,900 too high at $127,422 instead of $125,529. It also deducted the $144 IRA despite the active-participant phaseout and capped educator expenses at $300."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the 2024 $29,200 standard deduction and skipped the 401(k) exclusion and the $3,000 capital-loss deduction in AGI ($165,502). It counted only $3,174 of nonrefundable CTC and no CDCC, when the full $4,400 CTC plus the $1,920 CDCC applies against $17,031 of tax."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave a rough guess of $6,900 without doing the bracket calculation. On the correct $125,529 taxable income, tax is $17,031.48 before $6,320 of credits ($4,400 CTC and $1,920 CDCC). Its figure understates tax after nonrefundable credits by about $3,800."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used outdated amounts for 2026: a $31,500 standard deduction, a $2,000-per-child CTC, the 20% CDCC rate, and no $2,000 non-itemizer charitable deduction. It also used guessed brackets and deducted the $144 IRA, giving $12,404 instead of subtracting $6,320 of credits from $17,031."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It counted the $128 of qualified dividends twice in income. It also used a $31,500 standard deduction, a $2,000-per-child CTC, and a 20% CDCC, and left out the $2,000 non-itemizer charitable deduction. Its own work came to $12,421, but it submitted $18,962."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It treated the CDCC as about $0 even though the $12,740 of childcare expenses gives a $1,920 credit (32% of $6,000), and it used the $2,000-per-child CTC instead of $2,200. Its pre-credit tax of about $19,650 is also overstated: the brackets on $125,529 of taxable income give $17,031."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It deducted $1,575 of auto loan interest, which needs a new, US-assembled qualifying vehicle that the facts do not establish, and missed OBBBA's $2,000 non-itemizer charitable deduction. It also deducted the $144 IRA and capped educator expenses at $300. Together these leave taxable income $318 too high, which accounts for the $69.90 gap."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It counted only $600 of the CTC as nonrefundable, treating the $1,700 refundable cap as if it limited use against tax. In fact the full CTC ($2,200 per child under OBBBA) is used against regular tax first. It also used the 20% CDCC rate, a $30,000 standard deduction, and 2025 brackets, and left out the $2,000 charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It estimated about $19,900 of pre-credit tax, then added a made-up AMT amount to reach $24,500 instead of subtracting the credits. The correct path is $17,031 of tax less the $4,400 CTC and $1,920 CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It used the 20% CDCC rate ($1,200) instead of OBBBA's 32% at this joint AGI ($1,920). It also replaced OBBBA's $2,000 non-itemizer charitable deduction with a $1,575 auto loan interest deduction, which the facts do not support."
-us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It counted only $600 of the CTC as nonrefundable instead of the full $4,400 and used the 20% CDCC rate. It also ignored the $3,000 capital-loss deduction (adding the $540 short-term gain instead) and used a $30,900 standard deduction instead of $32,200 plus the $2,000 charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset law (personal exemptions, pre-2018 brackets, and a $1,000 CTC phased down to $750) instead of OBBBA's permanent extension with the $32,200 standard deduction and $2,200 CTC. It also took the employer-paid insurance premiums out of wages, pulling AGI down to $134,987."
-us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied TCJA-sunset law with $21,200 of personal exemptions, itemized deductions, and a $2,000 total CTC. It also excluded employer-sponsored premiums from wages, when 2026 OBBBA law calls for the $32,200 standard deduction, the $2,000 charitable deduction, the $4,400 CTC, and the 32% CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Its AGI was nearly right, but it used a $30,700 standard deduction instead of $32,200 and left out the $2,000 non-itemizer charitable deduction. It also applied the pre-OBBBA $2,000-per-child CTC and the 20% CDCC instead of $4,400 and $1,920."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules: personal exemptions, itemized deductions, a $1,000-per-child CTC, and a 20% CDCC. 2026 law under OBBBA keeps the $32,200 standard deduction, adds the $2,000 charitable deduction, and gives a $2,200-per-child CTC and a 32% CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It left out the $3,000 capital-loss deduction (AGI $164,481) and did not subtract the $4,400 CTC or the $1,920 CDCC. Its $17,822 is above even the correct pre-credit tax of $17,031."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset (personal exemptions, itemizing, and a phased-down $700 CTC) and took employer-sponsored premiums out of wages, giving AGI of $135,169. Under OBBBA, 2026 uses the $32,200 standard deduction, the $2,000 charitable deduction, and the full $4,400 CTC on AGI of $159,729."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied pre-TCJA 2026 law with $20,200 of personal exemptions and a phased-out $750 CTC. It also cut AGI to about $135,000 by excluding employer premiums from wages, instead of using OBBBA's $32,200 standard deduction, $4,400 CTC, and 32% CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no derivation. Its $11,463 is $752 above the correct figure, about the $720 gap between a 20% CDCC ($1,200) and OBBBA's 32% rate ($1,920) at this joint AGI."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $11,105 is $394 above the correct figure, close to the $440 of tax from leaving out OBBBA's $2,000 charitable deduction for non-itemizers at the 22% rate."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It said it used itemized deductions and personal exemptions, which means it applied TCJA-sunset law. Under OBBBA, 2026 law uses the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, with no personal exemptions."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $10,817 is $106 above the correct figure, which equals about $480 of extra taxable income at the 22% rate. Its explanation gives no credit amounts."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"It gave no numeric derivation. Its $14,256 is $3,545 above the correct figure, which means it did not apply the full $6,320 of nonrefundable credits ($4,400 CTC and $1,920 CDCC) against $17,031 of tax, or it overstated taxable income."
-us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It ignored the 401(k) exclusion and the capital-loss limit and applied a made-up $15,600 CTC. It also computed the CDCC for only one child and claimed a Saver's Credit that is unavailable at about $160,000 of joint AGI. The correct credits are $4,400 CTC plus $1,920 CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It counted only $800 of the CTC as nonrefundable, when the full $4,400 is used against regular tax. It also used the 20% CDCC instead of 32% and left out the $2,000 non-itemizer charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It gave no computation. Its $17,278 is above the correct pre-credit tax of $17,031, so it subtracted essentially none of the $4,400 CTC or $1,920 CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It gave no computation. Its $14,696 is about $4,000 above the correct figure, so it did not fully subtract the $4,400 CTC and $1,920 CDCC from $17,031 of tax."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It used the 20% CDCC rate ($1,200) instead of the OBBBA 32% rate ($1,920) and left out the $2,000 non-itemizer charitable deduction. Together these add about $1,160 of tax."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It applied a $1,200 CDCC at the old 20% rate instead of $1,920 at OBBBA's 32% rate for joint AGI of $159,729. Its implied pre-credit tax of $16,789 is also below the correct $17,031."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It left out the $2,000 non-itemizer charitable deduction (taxable $127,528 instead of $125,529) and used the 20% CDCC ($1,200) instead of OBBBA's 32% ($1,920)."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used the pre-OBBBA $2,000-per-child CTC ($4,000 instead of $4,400) and the 20% CDCC ($1,200 instead of $1,920). Its taxable income of about $128,000 also leaves out the $2,000 charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It used the 20% CDCC ($1,200) instead of 32% ($1,920) and left out the $2,000 non-itemizer charitable deduction. Its implied pre-credit tax of $18,360 is also overstated relative to the 2026 MFJ brackets."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It left out OBBBA's $2,000 charitable deduction for non-itemizers (taxable $127,384) and applied the 20% CDCC ($1,200) instead of the 32% rate ($1,920)."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Everything matched except the CDCC. It applied the OBBBA phase-down at 1 point per $2,000 over $150,000, giving 30% and $1,800, when joint returns step down per $4,000. The correct rate is 32% and the credit is $1,920."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,"It gave no numeric derivation. Its $12,545.56 is $1,834 above the correct figure, consistent with the 20% CDCC ($720 short), the missing $2,000 non-itemizer charitable deduction ($440), and an overstated bracket computation."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave an unsupported round estimate of $14,000. The correct calculation is $17,031 of tax on $125,529 of taxable income less $6,320 of nonrefundable credits ($4,400 CTC and $1,920 CDCC)."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied restored pre-2018 law (personal exemptions, 2017-style brackets, and a fully phased-out $1,000 CTC) instead of OBBBA's permanent rules, so it lost the $4,400 CTC. It also did not use the $32,200 standard deduction plus the $2,000 charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed the TCJA sunset ($21,200 of personal exemptions, 10/15/25% brackets, and a $1,000 CTC phased out above $110,000) and itemized. OBBBA law gives a $2,200-per-child CTC with no phaseout below $400,000 and a 32% CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used post-TCJA-expiration law (personal exemptions, 10/15/25% brackets, CTC fully phased out) instead of OBBBA's extension. That dropped the $4,400 CTC, cut the CDCC rate to 20%, and left out the $2,000 non-itemizer charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied pre-TCJA 10/15/25% brackets and a $2,000 CTC phased down to $751, and took employer premiums out of wages, giving AGI of $134,987. 2026 OBBBA law gives AGI of $159,729, the $32,200 standard deduction, the $4,400 CTC, and the 32% CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It used the 20% CDCC ($1,200) instead of OBBBA's 32% ($1,920) and left out the $2,000 non-itemizer charitable deduction, so tax ends up about $1,200 too high."
-us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no answer or explanation for this variable, so there was no value to score."
-us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no answer or explanation for this variable, so there was no value to score."
-us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed the charitable deductions wipe out the liability. But itemizing ($12,621 charity plus capped SALT) is worth less than the $32,200 standard deduction, and only $2,000 of charity is deductible on top of it. Taxable income stays at $125,529, with $10,711 of tax after credits."
-us,scenario_099,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It took an $8,060 qualified-overtime deduction even though no overtime compensation is listed as an input. It also used the 20% CDCC ($1,200) instead of 32% ($1,920) and left out the $2,000 non-itemizer charitable deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"It used the 20% CDCC ($1,200) instead of OBBBA's 32% ($1,920) and left out the $2,000 charitable deduction for non-itemizers (taxable $127,566 instead of $125,529)."
-us,scenario_099,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It used the 20% CDCC ($1,200 instead of $1,920), left out the $2,000 non-itemizer charitable deduction, and deducted the $144 IRA. It also computed tax with 2025 bracket breakpoints ($2,385 + $8,772) instead of the 2026 ones ($24,800 and $100,800)."
-us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It deducted the full $6,478 capital loss instead of the $3,000 limit and used a $32,600 standard deduction, a $2,000-per-child CTC, and a 20% CDCC. Its own work came to $11,730, but it submitted $24,711.50."
-us,scenario_099,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It never subtracted the $4,400 child tax credit and took off only a token CDCC amount. It also built AGI from an invented $4,160 educator deduction and skipped the $3,000 capital-loss deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"It used the old 20% CDCC rate ($1,200) instead of the 2026 enhanced 32% rate ($1,920), took an unsupported $1,575 auto-loan-interest deduction, and used a $32,600 standard deduction instead of $32,200. It then reported $12,103, which does not match its own $10,965 result."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"Its credits ($4,400 CTC plus $1,920 CDCC) are correct, but it left out the $2,000 charitable deduction that non-itemizers get on top of the $32,200 standard deduction. That made taxable income $127,422 instead of $125,529, about $417 too much tax at 22%. It also deducted the $144 traditional IRA, which the active-participant phaseout makes nondeductible."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It took a $7,272.73 overtime-premium deduction and a $1,575 auto-loan-interest deduction that the facts don't support. No qualified overtime compensation or qualifying new vehicle is listed, so neither applies. That understated taxable income at $116,536 versus $125,529, even though its CTC and 32% CDCC were right."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It built AGI of $165,502 without subtracting the $3,087 traditional 401(k) deferrals or the $3,000 capital-loss deduction, and used a $29,200 standard deduction instead of $32,200 plus the $2,000 charitable deduction. It also applied only $3,174 of CTC and no CDCC, rather than $4,400 CTC plus $1,920 CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave a rough $6,900 with no bracket computation. The correct calculation is $17,031.48 of tax on $125,529 of taxable income, minus $6,320 of credits ($4,400 CTC plus $1,920 CDCC), which gives $10,711. Its figure overstates credits or understates pre-credit tax by about $3,800."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It applied pre-OBBBA credit amounts: $2,000 per child CTC ($4,000) instead of $2,200 ($4,400), and the 20% CDCC rate ($1,200) instead of 32% ($1,920). It also used a $31,500 standard deduction with no $2,000 non-itemizer charitable deduction, taxed qualified dividends at 0%, and deducted the nondeductible IRA."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It submitted $18,962, which contradicts its own $12,421 result. That result was already wrong: it counted the $128 qualified dividends twice, used a $31,500 standard deduction with no $2,000 charitable deduction, and applied a $4,000 CTC and 20% CDCC instead of $4,400 and 32%."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It treated the CDCC as about zero instead of $1,920 (32% of the $6,000 expense cap) and used a $4,000 CTC instead of $4,400. It also overstated pre-credit tax at $19,650 on about $127,200 of taxable income, versus roughly $17,000, and missed the $2,000 non-itemizer charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It took a $1,575 auto-loan-interest deduction without any qualifying new-vehicle fact, and left out the $2,000 charitable deduction for non-itemizers. It also deducted the $144 traditional IRA despite the active-participant phaseout and capped educator expenses at $300 instead of allowing $337.50. Taxable income came out at $125,847 instead of $125,529, though its $6,320 of credits were correct."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"It treated only $300 per child of the CTC as nonrefundable ($600 total). In fact the whole CTC ($2,200 per child, $4,400 total) offsets liability first, and the refundable ACTC only matters when tax runs out. It also used the 20% CDCC rate instead of 32%, a $30,000 standard deduction instead of $32,200, and no $2,000 charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It estimated about $19,900 of pre-credit tax, then raised the after-credit figure to $24,500 by citing AMT and itemized-deduction add-backs that don't apply. Nonrefundable credits can only lower liability. It also used the 20% CDCC and a $4,000 CTC instead of $1,920 and $4,400."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,credit_phaseout,False,"It used the 20% CDCC rate ($1,200) instead of the enhanced 32% rate ($1,920) that applies at AGI about $159,729. It also took an unsupported $1,575 auto-loan-interest deduction in place of the $2,000 non-itemizer charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It added the $540 short-term gain as income without netting it against the $7,018 long-term loss, so it missed the $3,000 capital-loss deduction. It used a $30,900 standard deduction with no $2,000 charitable deduction, counted only $600 of CTC as nonrefundable instead of $4,400, and used a 20% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied TCJA-sunset law, with $20,244 of personal exemptions, itemized deductions, old-law brackets, and a phased-down $750 CTC. OBBBA made the $32,200 standard deduction, the TCJA brackets, and the $2,200 per child CTC permanent. It also wrongly subtracted employer-paid ESI premiums from wages, lowering AGI to $134,987."
+us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed TCJA expired and used $21,200 of personal exemptions and itemized deductions. OBBBA kept the $32,200 standard deduction and the $2,200 per child CTC with no exemptions. It also cut AGI to about $135,069 by treating employer ESI premiums as a Section 125 wage reduction, and it used a $2,000 CTC and 20% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"It applied a $2,000 per child CTC ($4,000) and the 20% CDCC ($1,200) instead of the 2026 amounts of $4,400 and $1,920. It also used a $30,700 standard deduction instead of $32,200 and left out the $2,000 non-itemizer charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It treated 2026 as a post-TCJA year, with $20,200 of personal exemptions, itemized deductions, and a $1,000 per child CTC. Under OBBBA there are no exemptions, the standard deduction is $32,200 plus a $2,000 non-itemizer charitable deduction, and the CTC is $2,200 per child. It also used a 20% CDCC instead of 32%."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It never applied the $3,000 capital-loss deduction (its AGI was $164,481) and never subtracted the $6,320 of nonrefundable credits ($4,400 CTC plus $1,920 CDCC). As a result, its $17,822 is essentially tax before credits."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied TCJA-sunset rules, with $21,200 of personal exemptions, itemizing, and a CTC phased down to $700. OBBBA kept the $32,200 standard deduction and the $2,200 per child CTC without exemptions. It also excluded ESI premiums from wages to get AGI of $135,169 instead of $159,729, and used a 20% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used pre-TCJA law, with $20,200 of personal exemptions, itemized deductions, and a $750 CTC after phaseout, instead of OBBBA's $32,200 standard deduction, $2,000 charitable deduction, and $4,400 CTC. It also put AGI at about $135,000 by improperly excluding employer insurance premiums, and used the 20% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,credit_phaseout,False,"It gave no derivation. Its $11,463 is $752 above the correct $10,711, which matches applying the 20% CDCC ($1,200) instead of the enhanced 32% rate ($1,920) on $6,000 of capped childcare expenses."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $11,105 is about $394 above the reference, which corresponds to about $1,800 of excess taxable income at 22%. That matches leaving out the $2,000 non-itemizer charitable deduction while still subtracting the full $6,320 of credits."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It explicitly used itemized deductions and personal exemptions for 2026. OBBBA made the TCJA framework permanent, so there are no personal exemptions, and the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction applies. It also missed the $4,400 CTC and the 32% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $10,817 is $106 above the correct $10,711.48, which is about $480 of excess taxable income at the 22% bracket after the full $6,320 of credits. It did not reach the $125,529.30 taxable income left after the $32,200 standard deduction and $2,000 charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"Its $14,256 implies only about $2,800 of nonrefundable credits against roughly $17,000 of pre-credit tax. The correct credits are $6,320: a $4,400 CTC ($2,200 per child, fully usable) plus a $1,920 CDCC (32% of $6,000)."
+us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"It invented a $15,600 CTC instead of $4,400, limited the CDCC to one child's $3,000 of expenses instead of $6,000 at 32%, and claimed a $1,404 saver's credit even though AGI of about $159,729 is far above the saver's-credit limit. It also skipped the 401(k) and capital-loss reductions to income."
+us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It counted only $800 of the CTC as nonrefundable, but the full $4,400 offsets liability before any refundable portion. It also used the 20% CDCC ($1,200) instead of 32% ($1,920) and left out the $2,000 non-itemizer charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"Its $17,278 is at or above the full pre-credit tax of $17,031.48. In effect it never subtracted the $6,320 of nonrefundable credits ($4,400 CTC plus $1,920 CDCC)."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"Its $14,696 implies only about $2,300 of nonrefundable credits against $17,031 of pre-credit tax. It left out most of the $4,400 CTC and $1,920 CDCC that this household can fully use."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It used the 20% CDCC ($1,200) instead of the enhanced 32% rate ($1,920) and left out the $2,000 non-itemizer charitable deduction. That made taxable income $127,566 instead of $125,529."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It subtracted a $1,200 CDCC (20% of $6,000) instead of the $1,920 that applies under the 2026 enhanced rate. At AGI $159,729, the 35% rate drops one point per $4,000 over $150,000, to 32%."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It used the 20% CDCC ($1,200) instead of 32% ($1,920) and left out the $2,000 non-itemizer charitable deduction. That gave taxable income of $127,528 instead of $125,529."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"It used the pre-OBBBA $2,000 per child CTC ($4,000) instead of $2,200 ($4,400), and the 20% CDCC ($1,200) instead of 32% ($1,920). It also left taxable income at about $128,000 without the $2,000 non-itemizer charitable deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,credit_phaseout,False,"It used the 20% CDCC ($1,200) instead of 32% ($1,920) and left out the $2,000 non-itemizer charitable deduction. Its implied pre-credit tax of $18,360 is also about $900 too high for the 2026 brackets (10% to $24,800, 12% to $100,800, then 22%)."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,credit_phaseout,False,"It used a $1,200 CDCC (20%) instead of $1,920 (32%) and left out the $2,000 non-itemizer charitable deduction. That gave taxable income of $127,384 instead of $125,529."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,credit_phaseout,False,"Its taxable income and CTC are correct, but it applied a 30% CDCC ($1,800) instead of 32% ($1,920). The enhanced rate drops from 35% by one point per $4,000 (or part) of AGI over $150,000 MFJ, so $9,729 of excess means three steps, to 32%."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,"Its $12,545.56 is about $1,834 above the reference. That gap is consistent with the old 20% CDCC ($720 short), a $2,000 per child CTC ($400 short), and leaving out the $2,000 non-itemizer charitable deduction (about $440)."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It gave a rough $14,000 without itemizing credits. The correct $17,031 of pre-credit tax minus the fully usable $4,400 CTC and $1,920 CDCC gives $10,711, so its figure undercounts credits by about $3,300."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied restored 2017-style law, with $21,200 of personal exemptions, 10%/15%/25% brackets, and a CTC fully phased out. OBBBA made the TCJA brackets, the $32,200 standard deduction, and the $2,200 per child CTC (phaseout starting at $400,000) permanent. It also used a 20% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It assumed TCJA expired, using personal exemptions, a $16,600 standard deduction, 10%/15%/25% brackets, and a $1,000 CTC phased out above $110,000. OBBBA set a $32,200 standard deduction plus a $2,000 charitable deduction and a $4,400 CTC that is unaffected at this AGI. It also used a 20% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It used post-TCJA-expiration law: $5,400 personal exemptions, itemizing, 10%/15%/25% brackets, and a fully phased-out CTC. Under OBBBA there are no exemptions, the $32,200 standard deduction plus the $2,000 charitable deduction applies, and the CTC is $4,400. It also applied the 20% CDCC instead of 32%."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied old-law 10%/15%/25% brackets and a CTC phased down to $751, using an AGI of $134,987 that improperly excluded employer insurance premiums. OBBBA keeps the TCJA brackets, the $32,200 standard deduction, and a $4,400 CTC. It also used a 20% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"It used the 20% CDCC ($1,200) instead of the enhanced 32% rate ($1,920) and left out the $2,000 non-itemizer charitable deduction. That left taxable income about $2,000 too high."
+us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no answer or explanation for federal_income_tax_before_refundable_credits.
+us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,It returned no answer or explanation for federal_income_tax_before_refundable_credits.
+us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It claimed charitable and other itemized deductions wipe out the tax. In fact itemized deductions fall short of the $32,200 standard deduction, so taxable income stays about $125,529. The resulting $17,031 of tax is only partly offset by $6,320 of credits."
+us,scenario_099,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It took an $8,060 overtime deduction, but no qualified overtime compensation is listed, so none applies. It also left out the $2,000 non-itemizer charitable deduction and used the 20% CDCC ($1,200) instead of 32% ($1,920)."
+us,scenario_099,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,credit_phaseout,False,"It used a $1,200 CDCC (20%) instead of the enhanced 32% rate ($1,920) and left out the $2,000 non-itemizer charitable deduction. That gave taxable income of $127,566 instead of $125,529."
+us,scenario_099,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It computed tax with 2025 bracket thresholds ($23,850 and $96,950) instead of the 2026 thresholds ($24,800 and $100,800). It also used the 20% CDCC ($1,200) instead of 32%, left out the $2,000 non-itemizer charitable deduction, and deducted the nondeductible traditional IRA."
+us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It submitted $24,711.50, which contradicts its own $11,730 computation. That computation was also wrong: it deducted the full $6,478 capital loss instead of the $3,000 limit, and used a $4,000 CTC and 20% CDCC instead of $4,400 and 32%."
+us,scenario_099,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It subtracted only the CDCC (at 20%) and never applied the $4,400 nonrefundable CTC. It also made up a $4,160 educator deduction, skipped the $3,000 capital-loss deduction, and used a $29,994 standard deduction instead of $32,200 plus the $2,000 charitable deduction."
us,scenario_099,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"Its own reasoning computed pre-credit federal tax of about $17,200 and stated that the $4,000 CTC reduces it to about $13,200, which under §24(d) leaves zero credit unused by the liability limitation, then abandoned that conclusion and reported $1,200 as an unexplained residual. It applied only the earned-income prong (15% × ($165,000 − $2,500) = $24,375) and never the controlling ceiling that the ACTC cannot exceed the CTC disallowed by tax liability. It also used the pre-OBBBA $2,000 per-child credit rather than the 2026 $2,200."
us,scenario_099,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,credit_phaseout,False,"It reported the statutory per-child refundable cap ($1,700 × 2 = $3,400) as an automatic refund for any filer with earnings above the $2,500 floor, skipping §24(d)'s requirement that the ACTC equals only the CTC remaining after the credit is applied against tax liability. With roughly $17,400 of federal income tax before credits, the full $4,400 of 2026 CTC is absorbed nonrefundably, so the refundable amount is $0. It also used the 2025 $1,700 cap instead of the 2026 indexed figure."
us,scenario_099,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"It correctly identified the OBBBA 2026 $2,200 per-child CTC and the $1,800 per-child refundable cap, then treated that cap as a guaranteed refund because earnings exceed the $2,500 threshold and AGI is under the $400,000 phase-out. The refundable portion is the lesser of the earned-income formula, the per-child cap, and the CTC left over after offsetting tax; on about $127,000 of taxable income the roughly $17,400 tax consumes all $4,400 of CTC nonrefundably, leaving $0."
us,scenario_099,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so the model performed no substantive computation to evaluate. The correct derivation gives $0: about $17,400 of pre-credit federal tax fully absorbs the $4,400 CTC under the §24(d) liability limitation, and $159,676 of income is far past the two-child MFJ EITC phase-out."
us,scenario_099,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No value or explanation was returned for federal_refundable_credits, so no substantive reasoning exists to fault. The correct derivation gives $0: the roughly $17,400 of federal tax before credits leaves no CTC unused for the ACTC, and EITC is zero at $159,676 of income."
us,scenario_099,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It invented a ""$300 Other Dependent Credit portion"" for two children aged 9 and 4 who are qualifying children; the §24(h)(4) other-dependent credit is $500, applies only to dependents who are not qualifying children, and is wholly nonrefundable, so it can never enter federal_refundable_credits. Its own explanation states that pre-credit tax absorbs the nonrefundable credits and no ACTC remains — a derivation that yields $0 — yet it submitted $600."
-us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"Compared about $166,694 of income to the federal 130% FPL free-meal cutoff and stopped there. It missed that California's universal school meals program gives the 9-year-old K-12 student free meals whatever the income, so PolicyEngine's tier is FREE."
-us,scenario_099,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"Concluded no free meals because income is above 130% FPL and there is no categorical eligibility. It ignored California's universal school meals program, which makes the K-12 child eligible for free meals regardless of either test."
-us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"Applied the federal 130% FPL income test and also invented an asset limit that school meals do not have. It never applied California's universal school meals program, which sets the tier to FREE for the enrolled 9-year-old despite the 5.03 FPG ratio."
-us,scenario_099,free_school_meals_eligible,claude-haiku-5.5,llm_error,state_local_rule,False,"Treated the household's high income as disqualifying. Under California's universal school meals program, income does not matter: the K-12 child gets free meals worth $1,119.60 a year."
-us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"Gave the free-meal cutoff as 185% FPL, which is actually the reduced-price limit (free is 130%), and denied eligibility on income. It also missed that California's universal school meals program makes the K-12 child eligible for free meals regardless of income."
-us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"Used only the federal 130% FPL test against $165,000+ of income. It never applied California's universal school meals program, under which PolicyEngine assigns the FREE tier to every enrolled K-12 student."
-us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,Stopped after the 130% FPL income test and the categorical eligibility check. It missed that California's universal school meals program overrides both and gives the 9-year-old free meals.
-us,scenario_099,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"Denied free meals because income exceeds 130% FPL. It failed to apply California's universal school meals program, which provides free meals to all enrolled K-12 students regardless of income."
-us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"Applied the federal NSLP 130% FPL test (about $42,900 for four) and concluded the household is ineligible. It overlooked California's universal school meals program, which sets the household's tier to FREE for the K-12 child."
-us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"Compared income to the federal 130%/185% FPL cutoffs only, and reversed the free and reduced-price labels. It never applied California's universal school meals program, which grants free meals to the enrolled 9-year-old whatever the income."
-us,scenario_099,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,Treated the 130% FPL free-meal limit as decisive for about $165k of income. California's universal school meals program makes the K-12 child eligible for free meals regardless of that limit.
-us,scenario_099,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Applied only the federal 130% FPL free-meal limit. It missed California's universal school meals program, under which PolicyEngine returns the FREE tier and $1,119.60 of support for the K-12 child."
-us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,Concluded ineligibility from income above 130% FPL alone. California's universal school meals program gives free meals to all enrolled K-12 students regardless of income.
-us,scenario_099,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Denied free meals on the 130% FPL income test. It did not apply California's universal school meals program, which makes the 9-year-old eligible for free meals regardless of income."
-us,scenario_099,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Used the federal 130% FPL cutoff as the only route to free meals. It missed California's universal school meals program, which sets the household's tier to FREE."
-us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,Treated an income threshold as controlling. California's universal school meals program gives free meals to the enrolled K-12 child with no income test.
-us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Applied the federal 130% FPL free-meal test and found the children ineligible. It missed that California's universal school meals program provides free meals to all K-12 students, so the household receives positive free-meal support."
-us,scenario_099,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Denied eligibility on income alone. Under California's universal school meals program, the K-12 child receives free meals regardless of the household's 5.03 FPG ratio."
-us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"Used an income eligibility threshold to deny free meals. It overlooked California's universal school meals program, which assigns the FREE tier to every enrolled K-12 student."
-us,scenario_099,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,Applied only the federal 130% FPL guideline. California's universal school meals program gives the enrolled 9-year-old free meals whatever the household income.
-us,scenario_099,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"Treated the income threshold as decisive. It missed California's universal school meals program, under which PolicyEngine returns $1,119.60 of free-meal support for the K-12 child."
-us,scenario_099,free_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,"Named California's universal meals program but set it aside and fell back on the federal 130% FPL test. PolicyEngine models the universal program and sets the tier to FREE, so the correct answer is eligible."
-us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"Compared AGI of about $156,588 to 130% FPL and concluded ineligibility. It never applied California's universal school meals program, which grants free meals to the K-12 child regardless of income."
-us,scenario_099,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"Applied the federal 130% FPL limit (about $43,000 for four) against about $160,000 of income. It missed that California's universal school meals program makes the enrolled 9-year-old eligible for free meals anyway."
-us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,Denied free meals because income is above the federal thresholds. California's universal school meals program gives free meals to all enrolled K-12 students with no income test.
-us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"Assumed wage income above federal thresholds rules out free meals. It ignored California's universal school meals program, which sets the household's tier to FREE for the K-12 child."
-us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"Applied the federal 130% FPL limit and found no categorical eligibility. It missed California's universal school meals program, under which PolicyEngine returns positive free-meal support whatever the income."
-us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,Treated income above the free-meal threshold as disqualifying. California's universal school meals program provides free meals to the enrolled K-12 child regardless of income.
-us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"Denied free meals on the income threshold alone. It overlooked California's universal school meals program, which assigns the FREE tier and $1,119.60 of support."
-us,scenario_099,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,Applied the federal free-meal income threshold. California's universal school meals program supersedes it and gives the K-12 child free meals.
-us,scenario_099,free_school_meals_eligible,gpt-6.1-terra,llm_error,state_local_rule,False,"Treated free meals as purely income-tested. In California, the universal school meals program provides free meals to all enrolled K-12 students without an income test."
-us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,"Concluded that high income rules out free meals. It missed California's universal school meals program, which makes the 9-year-old eligible regardless of income."
-us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"Applied the 130% FPL limit plus the categorical eligibility check and returned 0. California's universal school meals program sets the tier to FREE, and PolicyEngine returns $1,119.60."
-us,scenario_099,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"Checked only the 130% FPL limit and SNAP/TANF/Medicaid direct certification. It never applied California's universal school meals program, which gives free meals to every enrolled K-12 student."
-us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"Compared income to the federal 130% FPL cutoff (about $43,000). It missed California's universal school meals program, which makes the K-12 child eligible for free meals regardless of income."
-us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"Applied the 130% FPL guideline and found no categorical eligibility. It missed California's universal school meals program, under which PolicyEngine returns positive free-meal support for the enrolled 9-year-old."
-us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"Did not return a value for free_school_meals_eligible, so there is no answer to evaluate. The correct value is 1 because California's universal school meals program gives free meals to the K-12 child."
-us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,Treated income limits as decisive for free meals. California's universal school meals program provides free meals to all enrolled K-12 students regardless of income.
-us,scenario_099,free_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,"Denied free meals because income is above 130% FPL and there is no SNAP/TANF eligibility. It missed California's universal school meals program, which overrides both tests."
-us,scenario_099,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,Applied only the federal 130% FPL free-meal limit. California's universal school meals program gives the enrolled K-12 child free meals regardless of that limit.
-us,scenario_099,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"Compared about $160k of income to the 130% FPL cutoff (about $43k) and concluded no support. It overlooked California's universal school meals program, under which PolicyEngine sets the tier to FREE."
-us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"Applied the federal 130% FPL threshold to about $165,000 of income. It missed that California's universal school meals program makes the K-12 child eligible for free meals whatever the income."
-us,scenario_099,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,"Treated the income threshold as controlling. California's universal school meals program provides free meals to the enrolled 9-year-old, giving positive free-meal support."
+us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"The model compared roughly $166,694 of income to the federal 130% FPL free-meal cutoff and the 185% reduced-price cutoff and stopped there. It never applied California's universal free school meals program, which gives the 9-year-old K-12 child free meals at any income level."
+us,scenario_099,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model checked only the federal 130% FPL income test and categorical eligibility. It ignored California's universal school meals program, which makes the enrolled K-12 child eligible for free meals regardless of income."
+us,scenario_099,free_school_meals_eligible,claude-fable-5.2,llm_error,state_local_rule,False,The model correctly put income at about 500% of poverty but treated the federal 130% limit as the only route to free meals. California's universal meals program overrides that limit and gives the K-12 child free meals.
+us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model applied the federal 130% FPL test and added asset limits, which school meal programs do not have. It missed that California provides free meals to all enrolled K-12 students under its universal meals program, so the 9-year-old qualifies."
+us,scenario_099,free_school_meals_eligible,claude-haiku-5.5,llm_error,state_local_rule,False,The model rejected eligibility on income alone and did not account for California's universal free school meals program. That program has no income test and covers the household's 9-year-old K-12 student.
+us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model treated 185% FPL as the free-meal threshold, but 185% is the reduced-price cutoff. More importantly, it missed California's universal meals program, which grants free meals to the K-12 child with no income test."
+us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model applied the generic federal 130% FPL income test to $165,000 of wages. It ignored California's universal free school meals program, which makes the enrolled 9-year-old eligible whatever the household earns."
+us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model concluded 'income above 130% FPL and no categorical eligibility' and left out the third route to free meals: California's universal meals program, which covers every K-12 student."
+us,scenario_099,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"The model used the federal 130% FPL cutoff as the deciding test. It missed California's universal school meals program, which gives the K-12 child free meals regardless of the household's 5.03 FPG ratio."
+us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model applied only the federal NSLP 130% FPL limit (about $42,900). It did not recognize that California's universal meals program provides free meals to all enrolled K-12 students, including this household's 9-year-old."
+us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"The model tested $165,000 against the federal 130%/185% poverty thresholds and found the household ineligible. It overlooked California's universal free school meals program, which makes the K-12 child eligible regardless of income."
+us,scenario_099,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model compared about $165,000 to the 130% FPL free-meal threshold and stopped. California's universal meals program removes that income test for enrolled K-12 students, so the 9-year-old qualifies."
+us,scenario_099,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model applied only the federal 130% FPL free-meal limit. It missed California's universal school meals program, which grants free meals to the K-12 child at any income."
+us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model's one-line test, 'Income > 130% FPL', ignores California's universal free school meals program. That program supersedes the federal income test and covers the household's enrolled 9-year-old."
+us,scenario_099,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model treated exceeding 130% FPL as disqualifying. California's universal meals program provides free meals to every K-12 student regardless of income, so the household qualifies."
+us,scenario_099,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,The model applied the federal 130% FPL income limit and ignored the California rule that gives all enrolled K-12 students free meals with no income test.
+us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model rejected eligibility on an income threshold. California's universal free school meals program has no income threshold and covers the household's K-12 child.
+us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied the federal 130% FPL test to the children. It missed California's universal meals program, which makes the 9-year-old K-12 student eligible for free meals regardless of household income."
+us,scenario_099,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model treated free school meals as purely income-tested. It did not apply California's universal meals program, which grants free meals to all enrolled K-12 students."
+us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"The model ruled the household out on an income threshold. California's universal school meals program has no income test, so the K-12 child receives free meals."
+us,scenario_099,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"The model used the federal 130% poverty guideline as the only eligibility test and ignored California's universal free meals program, which covers the 9-year-old K-12 student."
+us,scenario_099,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"The model applied an income threshold to a California household. Under California's universal meals program, every enrolled K-12 student gets free meals regardless of income."
+us,scenario_099,free_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,"The model named California's universal meals program but still let the federal 130% FPL test decide the answer. The universal program supersedes the income test, which makes the K-12 child eligible for free meals and the output 1."
+us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model compared an estimated AGI of about $156,588 to 130% FPL. It missed that California's universal school meals program provides free meals to the enrolled K-12 child with no income test."
+us,scenario_099,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model applied the federal 130% poverty cutoff (about $43,000) and stopped. It ignored California's universal free school meals program, which covers the household's 9-year-old regardless of income."
+us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,The model treated income above the free-meal thresholds as disqualifying. California's universal meals program gives all enrolled K-12 students free meals regardless of income.
+us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed free meals require an income-threshold test or an indicated eligibility fact. California's universal meals program makes the 9-year-old K-12 child eligible automatically, with no income limit."
+us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model checked the federal 130% poverty limit and categorical eligibility and found neither met. It never considered California's universal free school meals program, which PolicyEngine applies to give the K-12 child free meals."
+us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,The model used an income threshold to deny free meals. California's universal school meals program sets the meal tier to FREE for the enrolled K-12 child regardless of income.
+us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model rejected eligibility because income exceeds the free-meal threshold. It missed California's universal meals program, under which the K-12 child receives free meals with no income test."
+us,scenario_099,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,The model applied the federal free-meal income threshold. California's universal school meals program overrides it and gives the K-12 student free meals.
+us,scenario_099,free_school_meals_eligible,gpt-6.1-terra,llm_error,state_local_rule,False,The model treated free school meals as income-tested only. In California the universal meals program is not income-tested and covers the household's enrolled 9-year-old.
+us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,"The model treated high income as disqualifying. Under California's universal free school meals program, every K-12 student gets free meals regardless of household income."
+us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model checked only the 130% FPL test and categorical eligibility. It missed California's universal meals program, which makes PolicyEngine return a positive free meal value of about $1,131 for the K-12 child."
+us,scenario_099,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"The model ruled out both the 130% income route and SNAP/TANF/Medicaid categorical certification. It missed the California universal meals pathway, which gives all enrolled K-12 students free meals."
+us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model applied the 130% FPL cutoff (about $43,000) and ignored California's universal free school meals program, under which the 9-year-old K-12 child receives free meals regardless of income."
+us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"The model checked the 130% FPL limit and the SNAP, TANF and foster-care categorical routes but did not apply California's universal meals program. That program gives the household's K-12 child free meals regardless of income."
+us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model gave no value or explanation for free_school_meals_eligible. The output is missing, not a substantive error. The correct answer is 1 because California's universal meals program covers the 9-year-old K-12 child."
+us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,The model applied federal income limits to deny free meals. California's universal school meals program has no income limit and covers the household's enrolled K-12 student.
+us,scenario_099,free_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,The model checked only the 130% FPL test and SNAP/TANF categorical eligibility. It missed that California's universal meals program makes the K-12 child eligible for free meals regardless of income.
+us,scenario_099,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"The model treated the 130% poverty guideline as binding. California's universal free school meals program removes the income test for all enrolled K-12 students, so the household qualifies."
+us,scenario_099,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model compared about $160,000 of income to the 130% FPL limit (about $43,000). It ignored California's universal meals program, under which PolicyEngine assigns the FREE tier to the K-12 child."
+us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"The model applied the federal 130% FPL threshold to this California household. It overlooked California's universal free school meals program, which gives the enrolled 9-year-old free meals regardless of income."
+us,scenario_099,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,"The model denied free meals on an income threshold. California's universal school meals program provides free meals to every K-12 student with no income test, so the household's 9-year-old qualifies."
us,scenario_099,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_099,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"Used a 1.2% CA SDI rate ($1,980) instead of the 2026 rate of 1.3% ($2,145). Then it submitted $14,118, which does not even match its own $14,602.50 total."
-us,scenario_099,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only federal FICA ($12,622.50). It left out California's mandatory employee SDI of 1.3% on $165,000 ($2,145), which the output definition explicitly includes."
-us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Said California has no mandatory employee payroll tax and so left out CA SDI. SDI is 1.3% of all wages in 2026, which adds $2,145 to the $12,622.50 of FICA."
-us,scenario_099,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"Wrongly said there is no mandatory CA state payroll tax and left out the $2,145 of SDI (1.3% of $165,000). It also misadded its own FICA figures, reporting $12,242 instead of $12,622.50."
-us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"Applied CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145). It then submitted $13,197.50, which contradicts its own recomputed $14,602.50."
-us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Calculated CA SDI at an outdated 1.1% rate, then dropped it entirely and reported federal FICA only ($12,622.50). The correct answer includes SDI at 1.3% ($2,145)."
-us,scenario_099,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Reported only Social Security and Medicare. It left out California's mandatory 1.3% employee SDI ($2,145) and also truncated Medicare to $2,392."
-us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Correctly treated CA SDI as uncapped and mandatory but used 1.1% instead of the 2026 rate of 1.3%. That made SDI $1,815 instead of $2,145, an understatement of $330."
-us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Wrongly called CA SDI voluntary. Its intermediate SDI figure used 1.1%, and it then submitted an unexplained $12,903.50. The correct SDI is 1.3% of $165,000, or $2,145, on top of $12,622.50 of FICA."
-us,scenario_099,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Applied CA SDI at about 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145), so the payroll tax total is $330 too low."
-us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"Subtracted the $24,597 of employer-sponsored insurance premiums from wages and taxed only $140,403. FICA and SDI should apply to the full $165,000 of gross wages. It also used a 1.1% SDI rate instead of 1.3%."
-us,scenario_099,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"Reduced the FICA and SDI wage base by the $24,597 of ESI premiums, to $140,403, when the full $165,000 of gross wages is taxable. It also used a 1.1% SDI rate instead of 1.3%."
-us,scenario_099,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Used a 1.2% CA SDI rate ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Used a 1.1% CA SDI rate ($1,815) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Included only Social Security and Medicare. It left out California's mandatory employee SDI of 1.3% on $165,000 ($2,145)."
-us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Subtracted $24,597 of ESI premiums from gross wages, leaving a $140,403 base for FICA and SDI, when the full $165,000 is taxable. It also applied 1.1% SDI instead of 1.3%."
-us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"Applied CA SDI at 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Counted only Social Security and Medicare and left out CA SDI ($2,145 at 1.3%). Its $12,963 also does not match the correct FICA total of $12,622.50, so the FICA arithmetic is wrong as well."
-us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"Said it included CA SDI, but $13,280 implies only about $657 of SDI on top of $12,622.50 of FICA. The correct SDI is 1.3% of $165,000, or $2,145."
-us,scenario_099,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,425.67 equals 8.85% (7.65% FICA plus 1.2% SDI) of $140,403. So it removed the $24,597 of ESI premiums from the wage base, which should be the full $165,000, and it used a 1.2% SDI rate instead of 1.3%."
-us,scenario_099,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"$14,103 implies about $1,480 of CA SDI (roughly 0.9% of wages) on top of $12,622.50 of FICA. The 2026 SDI rate of 1.3% gives $2,145."
-us,scenario_099,payroll_tax,gemini-4-argon,llm_error,thresholds_rates,False,"Applied a 1.1% CA SDI rate ($1,815) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"Left out CA SDI entirely. Its Social Security figures were wrong: the wages are under the wage base, so the tax is simply 6.2% × $165,000 = $10,230. It ended with $12,632.40 instead of $12,622.50 FICA plus $2,145 SDI."
-us,scenario_099,payroll_tax,glm-5.3,llm_error,thresholds_rates,False,"Correctly treated CA SDI as mandatory and uncapped but used 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Treated the payroll tax as federal FICA only and left out California's mandatory 1.3% employee SDI ($2,145)."
-us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Counted FICA only, leaving out the $2,145 of CA SDI. Its FICA figure of $13,577 is also wrong, since 7.65% of $165,000 is $12,622.50."
+us,scenario_099,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"Used 1.2% for CA SDI instead of the 2026 rate of 1.3%, which gives $1,980 instead of $2,145. It then submitted $14,118, which does not match even its own $14,602.50 total."
+us,scenario_099,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"Counted only employee FICA ($12,622.50) and left out California's mandatory employee SDI: 1.3% of $165,000, or $2,145."
+us,scenario_099,payroll_tax,claude-fable-5.2,llm_error,payroll_tax_base,False,"Used the correct 1.3% SDI rate but subtracted $3,088 of traditional 401(k) deferrals from the wage base. Elective deferrals stay subject to Social Security, Medicare and SDI, so all three taxes apply to the full $165,000."
+us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"Wrongly concluded that California has no mandatory employee payroll tax, so it left out the 1.3% employee SDI ($2,145) and reported FICA only."
+us,scenario_099,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"Wrongly stated that California has no mandatory employee payroll tax, leaving out $2,145 of SDI. It also mis-added its own FICA figures: $10,230 + $2,393 is $12,623, not $12,242."
+us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"Used a 1.2% CA SDI rate instead of 1.3%, which gives $1,980 instead of $2,145. It then submitted $13,197.50, which does not match its own recomputed $14,602.50."
+us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"Computed CA SDI at the outdated 1.1% rate and then dropped it entirely, treating payroll tax as federal FICA only. The output definition includes mandatory state employee payroll taxes, which here is $2,145 of SDI at 1.3%."
+us,scenario_099,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"Reported only Social Security and Medicare (with Medicare rounded down to $2,392) and left out California's mandatory 1.3% employee SDI of $2,145."
+us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"Correctly included uncapped CA SDI but used the 1.1% rate instead of the 2026 rate of 1.3%. This understates SDI by $330 ($1,815 instead of $2,145)."
+us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"Went back and forth on whether to include CA SDI, using 1.1% when it did. It then submitted $12,903.50, which matches none of its own computations. The correct total is FICA $12,622.50 plus SDI at 1.3% ($2,145)."
+us,scenario_099,payroll_tax,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"Applied CA SDI at 1.1% instead of the 2026 rate of 1.3%, giving $1,815 instead of $2,145. That leaves a total $330 too low."
+us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"Subtracted the $24,597 in ESI premiums, cutting the wage base to $140,403, when all payroll taxes apply to the full $165,000 in gross wages. It also used a 1.1% SDI rate instead of 1.3%."
+us,scenario_099,payroll_tax,deepseek-v4-pro-0813,llm_error,payroll_tax_base,False,"Applied every payroll tax to $140,403 after deducting ESI premiums instead of to the full $165,000 in gross wages. It also priced CA SDI at 1.1% instead of 1.3%."
+us,scenario_099,payroll_tax,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"Used a 1.2% CA SDI rate instead of the 2026 rate of 1.3%. This gives $1,980 of SDI instead of $2,145, so the total is $165 too low."
+us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"Applied CA SDI at 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145), understating the total by $330."
+us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Included only employee Social Security and Medicare. It left out California's mandatory employee SDI: 1.3% of $165,000, or $2,145."
+us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"Subtracted $24,597 in ESI premiums from wages before applying FICA and SDI, when the base is the full $165,000. It also used 1.1% for SDI instead of 1.3%."
+us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"Used a 1.1% CA SDI rate instead of the 2026 rate of 1.3%, which gives $1,815 instead of $2,145."
+us,scenario_099,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Counted only Social Security and Medicare, leaving out California's 1.3% employee SDI ($2,145). Its $12,963 also overstates even FICA alone, which is $12,622.50."
+us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"Included CA SDI but greatly understated it. Its total implies about $657.50 of SDI, while 1.3% of the full $165,000 is $2,145."
+us,scenario_099,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,425.67 is exactly 8.85% of $140,403. That means it subtracted ESI premiums from the wage base and used a 1.1% SDI rate, when the correct calculation is 7.65% FICA plus 1.3% SDI on the full $165,000."
+us,scenario_099,payroll_tax,gemini-3.8-flash,llm_error,thresholds_rates,False,"Included CA SDI but at too low an effective amount. Its total implies about $1,480 of SDI instead of $2,145 (1.3% of the uncapped $165,000)."
+us,scenario_099,payroll_tax,gemini-4-argon,llm_error,thresholds_rates,False,"Used the 1.1% CA SDI rate instead of the 2026 rate of 1.3%, understating SDI by $330."
+us,scenario_099,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"Left out California's mandatory 1.3% employee SDI ($2,145) entirely. It also garbled the FICA arithmetic, applying 6.2% to wage-base figures instead of to actual wages of $165,000, and submitted $12,632.40 instead of $12,622.50."
+us,scenario_099,payroll_tax,glm-5.3,llm_error,thresholds_rates,False,"Correctly applied uncapped CA SDI to gross wages but at 1.2% instead of the 2026 rate of 1.3%. That gives $1,980 instead of $2,145."
+us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"Treated FICA as the only employee payroll tax and left out California's mandatory 1.3% SDI ($2,145)."
+us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"Left out California's 1.3% employee SDI. It also mis-summed FICA, reporting $13,577 when 7.65% of $165,000 is $12,622.50."
us,scenario_099,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"Estimated CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"Stated an SDI rate of about 1.2% but submitted $14,312.40, which implies only about $1,690 of SDI. Applying the 2026 rate of 1.3% to $165,000 gives $2,145."
+us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"Said it was using about 1.2% CA SDI, but its total implies only about $1,690 of SDI. The correct amount is 1.3% of $165,000, or $2,145."
us,scenario_099,payroll_tax,gpt-5.6-sol,llm_error,thresholds_rates,False,"Used a 1.2% CA SDI rate ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"Used about 1.2% for CA SDI ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Counted only Social Security and Medicare and left out California's mandatory 1.3% employee SDI ($2,145)."
-us,scenario_099,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"Applied CA SDI at 1.1% ($1,815) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"Estimated CA SDI at about $1,980 (1.2%) instead of $2,145 (1.3% of uncapped wages)."
+us,scenario_099,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Counted only employee Social Security and Medicare, leaving out California's mandatory 1.3% employee SDI of $2,145."
+us,scenario_099,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"Applied CA SDI at 1.1% instead of the 2026 rate of 1.3%, understating SDI by $330."
us,scenario_099,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"Used a 1.2% CA SDI rate ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,grok-4.7,llm_error,thresholds_rates,False,"Used a 1.2% CA SDI rate ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"Removed $24,597 of ESI premiums from FICA wages instead of taxing the full $165,000 of gross wages. It also left out California's mandatory 1.3% employee SDI ($2,145)."
-us,scenario_099,payroll_tax,inkling,llm_error,thresholds_rates,False,"Used a 1.2% CA SDI rate ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there is no answer to score. The correct total is $12,622.50 FICA plus $2,145 CA SDI."
-us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there is no answer to score. The correct total is $12,622.50 FICA plus $2,145 CA SDI."
-us,scenario_099,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"Correctly computed $6,502.50 + $6,120 = $12,622.50 of employee FICA, then submitted $25,290, roughly double, as if the employer share were included. It also left out CA SDI ($2,145)."
-us,scenario_099,payroll_tax,muse-spark-1.3,llm_error,thresholds_rates,False,"Used about 1.2% for CA SDI ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,muse-wonder,llm_error,thresholds_rates,False,"Applied CA SDI at 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
-us,scenario_099,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"Said no employee-side state payroll tax applies and left out California's mandatory SDI. SDI is 1.3% of $165,000, or $2,145, on top of $12,622.50 of FICA."
-us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Acknowledged that CA SDI is mandatory and included in the definition, then dropped it and reported FICA only. Its SDI estimate also used an outdated 1.1% rate with a wage cap, instead of the 2026 rate of 1.3% with no cap ($2,145)."
-us,scenario_099,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Applied CA SDI to the head's wages only and got the arithmetic wrong: 1.2% of $85,000 is $1,020, not $472.20. It left out the spouse's SDI. The correct SDI is 1.3% of the combined $165,000, or $2,145."
+us,scenario_099,payroll_tax,grok-4.7,llm_error,thresholds_rates,False,"Applied uncapped CA SDI at 1.2% instead of 1.3%, which understates the total by $165."
+us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"Subtracted $24,597 in ESI premiums from FICA wages instead of applying FICA to the full $165,000. It also left out California's mandatory 1.3% employee SDI entirely."
+us,scenario_099,payroll_tax,inkling,llm_error,thresholds_rates,False,"Used 1.2% for CA SDI ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there was no answer to score against $14,767.50."
+us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"No payroll_tax value or explanation was returned, so there was no answer to score against $14,767.50."
+us,scenario_099,payroll_tax,minimax-m3,llm_error,other,False,"Computed per-person FICA of $6,502.50 and $6,120, which sum to $12,622.50, then submitted roughly double that ($25,290). It also left out California's 1.3% employee SDI of $2,145."
+us,scenario_099,payroll_tax,muse-spark-1.3,llm_error,thresholds_rates,False,"Estimated CA SDI at about 1.2% ($1,980) instead of the 2026 rate of 1.3% ($2,145)."
+us,scenario_099,payroll_tax,muse-wonder,llm_error,thresholds_rates,False,"Applied CA SDI at 1.2% instead of 1.3%, giving $1,980 instead of $2,145."
+us,scenario_099,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"Asserted that no employee-side state payroll tax applies. California's mandatory employee SDI applies at 1.3% of all wages ($2,145)."
+us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"Recognized that mandatory CA SDI belongs in the total, using the outdated 1.1% rate and a wage cap that no longer exists, but then submitted the FICA-only $12,622.50. The correct SDI is 1.3% of the full $165,000, or $2,145."
+us,scenario_099,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"Applied CA SDI only to the head and got the arithmetic wrong: 1.2% of $85,000 is $1,020, not $472.20. It also missed that SDI applies to the spouse's wages, at a 1.3% rate on all $165,000."
us,scenario_099,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -8612,55 +8706,56 @@ us,scenario_099,spouse_medicaid_eligible,kimi-k3,parse_contract_failure,missing_
us,scenario_099,spouse_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,spouse_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It correctly reached $138,791 of taxable income and $1,256 of exemption credits. However, it overstated bracket tax at about $6,940 instead of $5,871.48. It then submitted $6,222, which does not match even its own net of about $5,684."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It added a medical deduction of about $4,406 by double-counting both $2,500 premium fields and including over-the-counter costs. Allowable out-of-pocket medical costs (about $10,000) fall below the engine's 7.5% floor of about $11,980, so there is no medical deduction. That error cut taxable income to $134,241 instead of $138,792 and pulled the tax down to $4,251."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It took an outdated $4,754 standard deduction instead of itemizing $21,275 of real estate tax and charity, and it subtracted unreimbursed employee expenses above the line. It also invented a CA child tax credit, treated the educator expense as a tax credit, and never applied the $1,256 of personal and dependent exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It used the standard deduction and about $150,000 of taxable income, but itemizing $21,274.50 brings taxable income down to $138,792.30. It then applied an unanchored bracket estimate and landed at $4,200 instead of $5,871.48 minus $1,256 in credits."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It allowed the educator expense, which California disallows, and estimated $1,520 of exemption credits instead of $1,256. It reached a net of about $4,680, then arbitrarily inflated that to $5,959 by 'adjusting for SDI', even though SDI is a payroll tax and not part of income tax."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It started from $165,000 of gross wages without subtracting the $3,088 of traditional 401(k) deferrals, which inflated CA AGI to about $163,010 and taxable income to $141,735. It then overstated bracket tax at about $7,973. Its $1,256 of exemption credits was right."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Taxable income of about $138,000 was right, but it put bracket tax at about $7,700 instead of $5,871.48. It also subtracted only about $450 of exemption credits, even though its own listed amounts total about $1,300 and the correct figure is $1,256."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It deducted the $144 of traditional IRA contributions, which are not deductible here because both spouses have employer plans and income is above the phase-out. It also understated bracket tax at $5,729 instead of $5,871.48 and overstated exemption credits at $1,288 instead of $1,256, ending at $4,441."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly derived $138,791 of taxable income. However, it applied about $450 of exemption credit to every person, including the two adults, whose personal credit is only about $153 each. That gave $1,800 of credits instead of $1,256, and its guessed brackets produced $5,989 of tax instead of $5,871.48."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It applied a federal-style $10,000 SALT cap that California does not have, and estimated taxable income at about $150,000 instead of $138,792. It then submitted its pre-credit tax of $6,800 without subtracting the exemption credits it had just listed."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It added a medical deduction above the 7.5% floor, bringing itemized deductions to about $25,700. Allowable out-of-pocket medical costs do not exceed that floor, so itemized deductions are only $21,274.50. This understated taxable income at about $134,000 instead of $138,792 and the tax at $4,240."
-us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It inflated CA AGI to $163,306 by not netting out the $3,088 of 401(k) deferrals, and it added a $2,752 medical deduction even though medical costs fall below the 7.5% floor. It also subtracted only about $1,000 of credits instead of the $1,256 of personal plus dependent exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It subtracted the roughly $24,600 of employer-sponsored insurance premiums from wages, even though these premiums are employer-paid and do not reduce wage income. It also added unreimbursed employee expenses to itemized deductions, but the 2% floor wipes them out. Taxable income fell to $112,436 instead of $138,792.30."
-us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It reduced CA AGI by the employer-sponsored insurance premiums, and it padded itemized deductions to $27,545 with a medical amount that falls below the 7.5% floor. Taxable income came out at $107,524 instead of $138,792.30."
-us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"CA AGI of $160,066 was correct, but it added a $4,395 medical deduction by double-counting the two premium fields and including over-the-counter costs. Allowable medical costs fall below the 7.5% floor, so taxable income is $138,792.30 rather than $134,396."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It itemized correctly but applied only about $1,180 of exemption credits. The two $153 personal credits plus the two $475 dependent credits total $1,256, and that shortfall left its result about $50 above $4,615.48."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $7,845 is above even the $5,871.48 of pre-credit bracket tax on $138,792.30 of taxable income. That is consistent with skipping the $21,274.50 itemized deductions, overstating the brackets, and never subtracting the $1,256 of exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It cut CA AGI to $135,169 by subtracting the employer-sponsored insurance premiums from wages, and it padded itemized deductions to $26,138 with medical costs and employee expenses that fall below their floors. It also used only about $600 of exemption credits instead of $1,256."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The correct path is $138,792.30 taxable, $5,871.48 tax, and $1,256 of credits. The model's $2,570 implies roughly $25,000 less taxable income than that, which matches subtracting the employer insurance premiums from wages or deducting medical costs that fall below the 7.5% floor."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $8,725 exceeds the $5,871.48 of pre-credit tax on the correct $138,792.30 of taxable income. That matches ignoring the $21,274.50 of itemized deductions, overstating the rates, and leaving out the $1,256 of exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $4,128 is $487 short of $5,871.48 minus $1,256 of credits. The gap equals about $6,000 of extra deductions at the 8% marginal rate, which matches counting a medical deduction even though allowable medical costs are below the 7.5% floor."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $4,056 is about $560 below the correct $4,615.48. That gap matches adding deductions the household does not qualify for, such as medical costs below the 7.5% floor or employee expenses below the 2% floor, on top of the $21,274.50 of real estate tax and charity."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $5,245 overstates the correct $5,871.48 tax minus $1,256 of credits by $630. That matches using inflated bracket tax or subtracting only part of the personal and dependent exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,other,False,"Its $5,892 is essentially the $5,871.48 pre-credit bracket tax, so it never subtracted the $1,256 of exemption credits. It also wrongly cited renter's and child care credits, both unavailable at about $160,000 of AGI, and unreimbursed business expenses that the 2% floor eliminates."
-us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used a $6,566 standard deduction and claimed charity was not itemizable, but $21,274.50 of real estate tax and charity exceeds the $11,412 standard deduction. It treated the $338 educator expense as a credit even though California disallows it, and it submitted $3,202.96, which contradicts its own stated $5,559."
-us,scenario_099,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It took the roughly $11,500 standard deduction instead of itemizing $21,274.50, which pushed taxable income to about $149,000. It also used only about $620 of exemption credits instead of $1,256 and claimed a CA dependent care credit that is unavailable above $100,000 of AGI."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $6,713 is above the $5,871.48 of pre-credit tax on the correct $138,792.30 of taxable income. That matches using the standard deduction instead of the $21,274.50 of itemized deductions and leaving out the $1,256 of exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"Its $4,931 is $316 above the correct $5,871.48 minus $1,256. That matches overstated 2026 bracket tax or only partial exemption credits on otherwise correct taxable income."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It cut CA AGI to $134,323 by excluding the health premiums from wages and deducting IRA contributions, even though neither reduces CA AGI here. It treated exemptions as deductions rather than credits, and its $6,882 tax does not match its own $101,811 taxable income."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used the California standard deduction even though $8,653.50 of real estate tax plus $12,621 of charity ($21,274.50) exceeds it. That overstated taxable income by about $10,000 at the 8–9.3% marginal rates and produced $5,489."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It had the right CA AGI and itemized deductions but understated the result at $4,382 instead of $4,615.48. That gap comes from underestimating 2026 bracket tax below $5,871.48 or overstating credits above the $1,256 of exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $4,250 is $365 below $5,871.48 minus $1,256. That matches underestimating 2026 MFJ bracket tax on $138,792.30 of taxable income or overstating the exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It itemized the right items but reached $5,138, which is $523 too high. That matches overstated bracket tax on $138,792.30 or a shortfall in the $1,256 of personal and dependent exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It deducted the $144 of traditional IRA contributions, which are fully phased out because both spouses have employer plans, so taxable income was $138,647 instead of $138,792.30. It also slightly understated bracket tax, ending at $4,566."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It followed the right structure but reached $4,460, $155 below $4,615.48. That matches underestimating 2026 bracket tax on $138,792.30 (the correct tax is $5,871.48) or overstating credits beyond $1,256."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It produced a rough guess of about $4,500 on '~165k income' without computing CA AGI of $160,066.80, $21,274.50 of itemized deductions, $5,871.48 of bracket tax, or $1,256 of exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It started from federal AGI of $159,728 without adding back the $337.50 educator expense that California disallows. It also overstated exemption credits at about $1,290 instead of $1,256 and rounded to $4,600."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Taxable income of $138,791 was correct, but its projected 2026 brackets gave $5,743 of tax instead of $5,871.48. It also used $1,246 of credits instead of $1,256, ending at $4,497."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It used CA AGI of $159,766 without adding back the $337.50 educator expense, which California does not allow. It also underestimated bracket tax on that income, ending at $4,427."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It shrank taxable income to $110,811 by deducting items that do not reduce CA taxable income, such as employer insurance premiums. It also said no nonrefundable credits apply, but the $1,256 of personal and dependent exemption credits applies at this income."
-us,scenario_099,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It subtracted only $576 of exemption credits instead of $1,256, leaving out most of the roughly $475-per-child dependent credits. It also used $138,491 of taxable income because it skipped the $337.50 educator add-back."
-us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No answer or explanation was returned for state_income_tax_before_refundable_credits, so there was no value to score."
-us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"No answer or explanation was returned for state_income_tax_before_refundable_credits, so there was no value to score."
-us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported $0 even though $138,792.30 of CA taxable income produces $5,871.48 of bracket tax. Only $1,256 of exemption credits offsets that, leaving $4,615.48."
-us,scenario_099,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It subtracted only about $632 of exemption credits instead of $1,256, which leaves out most of the dependent credits. It also overstated bracket tax at about $6,001 and skipped the $337.50 educator add-back."
-us,scenario_099,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It used about $149,000 of taxable income from the standard deduction instead of itemizing $21,274.50. It also subtracted about $240 of CA dependent care credit, which is unavailable above $100,000 of AGI."
-us,scenario_099,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It assumed California requires the standard deduction because the federal standard deduction was claimed. California lets filers itemize independently, so the $21,274.50 of real estate tax and charity applies and taxable income is $138,792.30, not about $147,800."
-us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It netted the full $6,478 capital loss instead of the $3,000 cap, did not subtract 401(k) deferrals from wages, and allowed the educator expense even though California disallows it. It also said no nonrefundable credits apply, but the $1,256 of exemption credits applies."
-us,scenario_099,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used a nonexistent $34,000 standard deduction instead of itemizing $21,274.50. It applied a renter's credit the household cannot get at this AGI and never subtracted the $1,256 of personal and dependent exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It correctly reached $138,791 of taxable income and $1,256 of exemption credits. But it put the MFJ bracket tax at about $6,940 instead of $5,871. It then reported $6,222, which does not even match its own $5,684 net figure."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It added a medical deduction of about $4,406, which is consistent with counting both premium fields plus OTC costs. Qualifying out-of-pocket medical expenses (about $10,000) fall below the 7.5% floor of about $12,005, so the medical deduction is zero. This cut taxable income to about $134,241 instead of $138,792."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It started from a CA AGI of $159,584 after deducting the educator expense, which California disallows, and a $144 IRA deduction, which phases out for these plan-covered filers. It also cut charity to $11,823 instead of the full $12,621. The result was $139,107 of taxable income instead of $138,792."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It used an invented $4,754 standard deduction instead of the $21,275 of itemized real estate tax and charity, which put taxable income at $155,605 instead of $138,792. It also replaced the $1,256 personal and dependent exemption credits with made-up child credits and an educator-expense 'benefit'."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,taxable_income_or_deductions,False,"It assumed about $150,000 of taxable income after the standard deduction. It never itemized the $8,654 of real estate tax plus the $12,621 of charity, which brings taxable income down to $138,792. Its $4,200 is a rough guess, not a computation from the brackets."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It wrongly treated the educator expense as deductible in California. It also overstated the exemption credits at about $1,520 instead of $1,256. Its own work reached about $4,680, but it then raised the answer to $5,959 by citing SDI, which is a payroll tax excluded from income tax."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It left the $3,088 of traditional 401(k) deferrals in wages, giving CA AGI of $163,010 and taxable income of $141,735. It then put the bracket tax at about $7,973, which wrongly places this income in the 9.3% bracket. The correct tax on $138,792 is $5,871."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Its taxable income of about $138,000 was right. But it overstated the bracket tax at about $7,700 instead of $5,871. It also miscounted the exemption credits as about $450, even though its own inputs total about $1,300; the correct figure is $1,256."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It subtracted a $144 IRA deduction that phases out for these covered filers. Its projected 2026 brackets gave $5,729 instead of $5,871. It also used $1,288 of exemption credits instead of $1,256 ($153×2 + $475×2)."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It correctly reached $138,791 of taxable income. But it valued each exemption credit at about $450, totaling $1,800. In fact the personal credit is about $153 and the dependent credit about $475, totaling $1,256, so it subtracted about $544 too much."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It put taxable income at about $150,000 by under-counting the itemized deductions and applying a federal-style $10k SALT cap that California does not have. It then subtracted none of the $1,256 of personal and dependent exemption credits, leaving its pre-credit $6,800 unchanged."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It included a medical deduction above the 7.5% floor, about $4.4k. Qualifying out-of-pocket medical costs (about $10,000) are below the roughly $12,005 floor; it reached $4.4k only by double-counting premiums and adding OTC costs. That cut taxable income to about $134k instead of $138,792."
+us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It overstated CA AGI at $163,306 instead of $160,067 and added a $2,752 medical deduction. That deduction does not exist because medical costs are below the 7.5% floor. It also took only about $1,000 of dependent credits, omitting the two $153 personal credits that bring the total to $1,256."
+us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"Its $112,436 of taxable income implies a CA AGI of about $135k. That is consistent with subtracting the roughly $24.6k of employer-sponsored premiums, which never enter W-2 wages. It also inflated itemized deductions to $22,551, consistent with deducting the $1,263 of employee expenses without the 2% AGI floor."
+us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its $107,524 of taxable income implies a CA AGI of about $135k, consistent with subtracting the roughly $24.6k of employer-paid ESI premiums from wages. It also claimed $27,545 of itemized deductions, adding medical expenses that fall below the 7.5% floor. The correct itemized total is $21,275."
+us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"Its AGI, real estate tax, charity, brackets and exemption credits were all essentially right. Its only error was a $4,395 medical deduction that double-counts the premium fields and includes OTC costs. Allowable medical expenses do not exceed 7.5% of AGI, so taxable income is $138,792, not $134,396."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used about $1,180 of exemption credits instead of $1,256 (2×$153 personal + 2×$475 dependent). Its implied pre-credit tax was about $5,845 instead of $5,871, which together put it about $50 above the correct $4,615."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no computation. Its $7,845 exceeds even the correct pre-credit bracket tax of $5,871 on $138,792 of taxable income. That is consistent with skipping the $21,275 of itemized deductions, overstating the rates, and omitting the $1,256 of exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It understated CA AGI at $135,169 instead of $160,067, consistent with subtracting the roughly $24.6k of employer-paid health premiums from wages. It inflated itemized deductions to about $26,138 with non-allowable medical and business expenses. It also counted only about $600 of exemption credits instead of $1,256."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The correct path is $138,792 of taxable income, $5,871 of tax, and $1,256 of credits. Its $2,570 instead implies taxable income of about $115k. That is consistent with removing the roughly $24.6k of employer ESI premiums from income, or adding a medical deduction below the 7.5% floor."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"It gave no computation. Its $8,725 is well above even the tax on income after only the $11,412 standard deduction (about $7,160 before credits). It therefore ignored the itemized deductions, overstated the CA rate schedule, and omitted the $1,256 of exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $4,128 is about $487 below the correct $4,615 ($5,871 bracket tax less $1,256 of credits). That gap matches about $6k of extra deductions at the 8% marginal rate, the pattern of claiming a medical deduction even though expenses are below the 7.5% floor."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $4,056 is about $559 below the correct $4,615. That matches about $7k of extra deductions at the 8% marginal rate, the pattern of adding a nonexistent medical deduction to the $21,275 of real estate tax and charity."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation. Its $5,245 is $630 above the correct $4,615. That is consistent with under-claiming the $21,275 of itemized deductions (about $7.9k too much taxable income at 8%) or omitting part of the $1,256 of exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"Its $5,892 essentially equals the pre-credit bracket tax of $5,871, so it never actually subtracted the $1,256 of personal and dependent exemption credits. It also wrongly listed unreimbursed employee expenses, which fall below the 2% floor, and renter's/childcare credits, which are unavailable above $100k AGI."
+us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It used a $6,566 standard deduction and claimed that charity does not exceed it. The $8,654 of real estate tax plus the $12,621 of charity ($21,275) clearly exceed the $11,412 CA standard deduction. It then subtracted the educator expense as if it were a credit, and its final $3,202.96 contradicts its own $5,559."
+us,scenario_099,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It took the roughly $11,500 standard deduction instead of itemizing $21,275, giving about $149k of taxable income instead of $138,792. It also used only about $620 of exemption credits instead of $1,256, and claimed a CA dependent care credit that is unavailable at AGI over $100k."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its $6,713 matches the bracket tax on income after only the standard deduction (about $148.7k taxable, about $6,700 of tax) with no exemption credits. The correct path itemizes $21,275 and subtracts $1,256 of exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It gave no figures. Its $4,931 is about $316 above the correct $4,615, which matches subtracting only the two $475 dependent credits from the $5,871 bracket tax. That shortcut omits the two $153 personal exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It cut CA AGI to $134,323 by excluding health premiums that are not part of W-2 wages, and treated exemptions as deductions rather than credits. It then reported $6,882 of tax on $101,811 of taxable income, while the CA MFJ schedule yields only about $3,080 on that amount."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It applied the CA standard deduction instead of itemizing the $8,654 of real estate tax plus the $12,621 of charity. That leaves about $148.7k of taxable income, and roughly $5.45k after the $1,256 of credits, instead of $138,792 and $4,615."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It correctly used $160,066 of AGI and the $21,275 of itemized deductions, giving $138,792 of taxable income. Its bracket tax less credits came out $233 short of $5,871 minus $1,256. That means its projected bracket thresholds or exemption-credit amounts were overstated."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It gave no figures. Its $4,250 is $365 below the correct $5,871 of bracket tax on $138,792 less $1,256 of exemption credits. That is consistent with overstated bracket thresholds or credits, or with a small extra deduction."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It correctly itemized the property tax and charity. Its $5,138 is $523 above the correct figure, which implies it subtracted only about $733 of exemption credits instead of $1,256 ($153×2 + $475×2), or overstated the bracket tax."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It deducted a $144 IRA contribution that is fully phased out for these plan-covered filers, giving $138,647 of taxable income. Its bracket estimate then fell about $38 short of the correct schedule, which yields about $5,860 on that income."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It gave no figures. Its $4,460 is $155 below the correct $5,871 of bracket tax on $138,792 less $1,256 of credits. That is consistent with bracket thresholds indexed too high or exemption credits overstated by about that amount."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It guessed '~4.5k' from gross income without computing CA AGI ($160,067), itemized deductions ($21,275), the bracket tax on $138,792 ($5,871), or the $1,256 of exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used federal AGI and never added back the $338 educator deduction that California disallows, giving $138,453 of taxable income instead of $138,792. It also used about $1,290 of exemption credits instead of $1,256, leaving it slightly low."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It correctly reached $138,791 of taxable income. But its 2026 brackets gave $5,743 of tentative tax instead of $5,871, and it used $1,246 of exemption credits instead of $1,256 ($153×2 + $475×2)."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It understated CA AGI by $300 at $159,766. Its bracket computation on $138,491 then fell about $164 short of the correct schedule (about $5,847 before the $1,256 of credits)."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It understated taxable income at $110,811 instead of $138,792, consistent with removing the employer-paid ESI premiums from wages. It also claimed that no nonrefundable credits apply, omitting the $1,256 of personal and dependent exemption credits, which do not phase out at this AGI."
+us,scenario_099,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"Its taxable income of $138,491 and bracket tax of about $5,866 were close. But it took only $576 of exemption credits instead of $1,256: the dependent credit is about $475 per child, not a personal-credit-sized amount."
+us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score against the $4,615.48 figure."
+us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score against the $4,615.48 figure."
+us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported $0 with no computation. This household has $138,792 of CA taxable income, owes $5,871 of bracket tax, and has only $1,256 of nonrefundable exemption credits to offset it."
+us,scenario_099,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It took only about $632 of exemption credits instead of $1,256 (2×$153 personal + 2×$475 dependent). It also overstated the bracket tax on its $138,491 of taxable income at about $6,001 instead of about $5,847."
+us,scenario_099,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It used about $149k of taxable income (the standard deduction) instead of itemizing $21,275 to reach $138,792. It overstated the bracket tax at $7,136. It also subtracted a $240 CA dependent care credit, which is unavailable at AGI over $100k."
+us,scenario_099,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It wrongly held that California forces the standard deduction because the federal standard deduction is claimed. California lets this couple itemize $21,275 of real estate tax and charity regardless of their federal choice. As a result it used $147,800 of taxable income instead of $138,792."
+us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It skipped the $3,088 of 401(k) deferrals, used the full $6,478 capital loss instead of the $3,000 cap, and deducted the educator expense that California disallows. It then put tax on $138,063 at about $4,300 instead of about $5,813, and claimed no nonrefundable credits apply, omitting the $1,256 of exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used a nonexistent $34,000 standard deduction instead of the $21,275 of itemized deductions. It overstated the bracket tax at $8,100 on $130,894. It also applied a renter's credit the couple is ineligible for at $160k AGI while omitting the $1,256 of exemption credits."
us,scenario_099,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_099,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -8771,57 +8866,58 @@ us,scenario_100,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model identi
us,scenario_100,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model computed both components correctly ($5,915 × 0.062 = $366.73 and $5,915 × 0.0145 = $85.78, totaling $452.51) and then discarded that result, submitting $477 on the invented rationale of 'rounding to annual benchmark.' No rounding, annualization, or state payroll-tax add-on applies here; the answer its own arithmetic supports is $452.48."
us,scenario_100,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below 185% of the poverty guideline as sufficient for reduced-price meals and ignored that income at or below 130% qualifies for free meals. At 44% of the guideline, reinforced by categorical eligibility, the household receives the mutually exclusive FREE tier rather than reduced-price support."
us,scenario_100,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly asserted that $5,915 was above 130% of the poverty guideline for a four-person household, even though the engine ratio is 44%. It therefore placed the household in the 130%-185% reduced-price band instead of the FREE tier, while also overlooking categorical eligibility for free meals."
-us,scenario_100,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"It counted only the $493/month of wages and left out the ~$505/month TANF cash income. It took no utility-allowance shelter deduction, as policyengine-us 2.60.0 does; the frozen reference's $453.64 excess shelter deduction comes from engine defect r30_snap_heat_and_eat_sua. It took a $94 excess medical deduction that policyengine-us 2.60.0 disallows because the head receives no qualifying disability benefit and has no qualifying veteran-status flag. The result was $89 of net income, against $691 on policyengine-us 2.60.0 (the frozen reference has $237.09). Its $785 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026)."
-us,scenario_100,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Its net income of $1,040/year comes from wages alone. It omitted the TANF cash income and applied an excess medical deduction that policyengine-us 2.60.0 disallows, because the head receives no SSI or other disability benefit. It also used a $214 standard deduction instead of $209. Taking no utility-allowance shelter deduction matches policyengine-us 2.60.0; the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua. Its flat $785 allotment matches the release's SNAP convention (c_snap_hold_fy2026), which holds the FY2026 maximum for all 12 months."
-us,scenario_100,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"It found the household eligible but never worked out net income or the benefit, and submitted an unexplained 'conservative' $1,908. policyengine-us 2.60.0 gives net income of $691 and $577/month, which is $6,924 for the year. The frozen reference's $237.09 net income and about $714–$734/month ($8,625.89) include the $453.64 shelter deduction from engine defect r30_snap_heat_and_eat_sua and the engine's projected FY2027 October–December allotment, and the output is excluded from scoring."
-us,scenario_100,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"It returned $0 with no valid disqualifying reason. Assets of $2,800 are within the limits, the household is categorically eligible through TANF, and on policyengine-us 2.60.0 net income of $691 still leaves a $577/month benefit."
-us,scenario_100,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Its working left out the ~$505/month TANF cash income. Taking no utility-allowance shelter deduction matches policyengine-us 2.60.0; the frozen reference's $453.64 excess shelter deduction comes from engine defect r30_snap_heat_and_eat_sua. It also applied a $141 medical deduction that double-counted the $564 premium and is not available, because the head is not a SNAP disabled member. It then cut its own $9,108 figure to $7,320 with no calculation behind the reduction."
-us,scenario_100,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"It used a $766 maximum allotment instead of the FY2026 $785, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months, and it computed net income from wages alone, without the TANF cash income. Taking no utility-allowance shelter deduction matches policyengine-us 2.60.0; the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua. It then lowered its $8,496 result to $8,316 with no supporting calculation."
-us,scenario_100,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"It estimated net income from wages, the earned income deduction and the standard deduction only. It never added the ~$505/month TANF cash income, so its rounded $8,300 does not match policyengine-us 2.60.0's $577/month ($6,924/year). Taking no utility-allowance shelter deduction matches that engine; the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua, and its $733.60 for Oct–Dec uses the engine's projected FY2027 allotment, which the release's SNAP convention (c_snap_hold_fy2026) replaces with FY2026 all year."
-us,scenario_100,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"It counted only wages ($493) and missed the ~$505/month TANF cash income; taking no utility-allowance shelter deduction matches policyengine-us 2.60.0, and the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua. It took a $94 medical deduction that policyengine-us 2.60.0 disallows because the head receives no SSI or SSDI, has no other qualifying benefit receipt, and has no qualifying veteran-status flag. Its net income of $92 compares with $691 on policyengine-us 2.60.0 (the frozen reference has $237.09), and it applied $785 for all 12 months."
-us,scenario_100,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"It treated the absence of rent as zero shelter cost, which matches policyengine-us 2.60.0; the frozen reference's $453.64 excess shelter deduction from the utility allowance comes from engine defect r30_snap_heat_and_eat_sua. It left the ~$505/month TANF cash income out of gross income and applied a $93.67 excess medical deduction that the engine disallows, because the head receives no disability benefit. With a $228 standard deduction instead of $209, its net income came to $72.67, against $691 on policyengine-us 2.60.0 (the frozen reference has $237.09)."
-us,scenario_100,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Its $13,700 is above the maximum possible annual allotment of $9,420 for three people ($785 × 12 under the release's SNAP convention, c_snap_hold_fy2026), and it does not equal its own $766 × 12 = $9,192. It also deducted the $20,644 employer-sponsored insurance premium as a medical expense and ignored the TANF cash income."
-us,scenario_100,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"It brought net income close to zero with a disabled-member medical deduction that does not apply, because the head receives no disability benefit. It also left out the ~$505/month TANF cash income. On policyengine-us 2.60.0, which takes no utility-allowance shelter deduction, net income is $691 and the contribution $208, which leaves $577/month. The frozen reference's $237.09 and $71.10 include a $453.64 shelter deduction from engine defect r30_snap_heat_and_eat_sua."
-us,scenario_100,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"It added the $710 overtime premium on top of $5,915 in wages, even though the stated wages already include overtime. It counted the $564 premium twice in a medical deduction the head does not qualify for, and it left out the TANF cash income; taking no utility-allowance shelter deduction matches policyengine-us 2.60.0, and the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua. It also used a flat $797 allotment instead of the FY2026 $785, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months."
-us,scenario_100,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"It correctly counted TANF as unearned income (estimating $458.54 against the actual ~$505) and took no utility-allowance shelter deduction, as policyengine-us 2.60.0 does (the frozen reference's $453.64 excess shelter deduction comes from engine defect r30_snap_heat_and_eat_sua), but it took a medical deduction that the engine disallows, because the head receives no disability benefit. Its net income came to $602, against $691 on policyengine-us 2.60.0, and its $768 maximum allotment is below the FY2026 $785 that the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months."
-us,scenario_100,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Its submitted $9,037.20 contradicts its own stated $7,808.48. That derivation understated TANF at $335/month and applied a $140.67 medical deduction that policyengine-us 2.60.0 disallows, because the head receives no disability benefit; taking no utility-allowance shelter deduction matches that engine, and the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua. It also used a $198 standard deduction and a $768 maximum allotment instead of $209 and the FY2026 $785 that the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months."
-us,scenario_100,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"It counted only wages, missing the ~$505/month TANF cash income. Taking no utility-allowance shelter deduction matches policyengine-us 2.60.0; the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua. It applied a $53.67 medical deduction that the engine disallows for a head without disability-benefit receipt, and it used an $805 allotment for all 12 months instead of the FY2026 $785, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months."
-us,scenario_100,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"It deducted $140/month of medical costs (counting the $564 premium twice) for a head who is not a SNAP disabled member. It also left out the TANF cash income; taking no utility-allowance shelter deduction matches policyengine-us 2.60.0, and the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua. Its net income of $54 compares with $691 on policyengine-us 2.60.0 (the frozen reference has $237.09), and its $781 maximum allotment is below the FY2026 $785."
-us,scenario_100,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"It gave a near-maximum benefit with no income calculation and ignored the ~$505/month TANF cash income. With that income, policyengine-us 2.60.0 gives net income of $691 after deductions, and the $208 contribution brings the benefit to $577/month."
-us,scenario_100,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"It said the household gets close to the maximum allotment, yet submitted $6,128 (about $511/month), far below the $785 maximum, with no calculation to support it. policyengine-us 2.60.0 gives $577/month ($6,924/year). The frozen reference's $713.90/month for Jan–Sep and $733.60 for Oct–Dec include the $453.64 shelter deduction from engine defect r30_snap_heat_and_eat_sua and the engine's projected FY2027 October–December allotment."
-us,scenario_100,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"It correctly counted TANF as unearned income and took no utility-allowance shelter deduction, as policyengine-us 2.60.0 does (the frozen reference's $453.64 excess shelter deduction comes from engine defect r30_snap_heat_and_eat_sua), but it applied a medical deduction that the engine disallows because the head receives no qualifying disability benefit, and it used a $768 maximum allotment instead of the FY2026 $785. Its implied net income was about $544, above the frozen reference's $237.09 and below the $691 policyengine-us 2.60.0 gives. Its benefit came to about $605/month, against $577/month on policyengine-us 2.60.0."
-us,scenario_100,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"It wrongly concluded that income and resource limits disqualify the household. Gross income is 45% of the poverty guideline, assets of $2,800 are within limits, and TANF gives categorical eligibility, so the benefit is positive: $577/month on policyengine-us 2.60.0."
-us,scenario_100,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"It counted only wages plus a disabled-member medical deduction. It missed the ~$505/month TANF cash income and applied a medical deduction that policyengine-us 2.60.0 disallows without SSI or SSDI receipt, so its net income ended up well below the $691 that engine gives. Taking no utility-allowance shelter deduction matches that engine; the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua."
-us,scenario_100,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Its submitted $9,216 contradicts the 9,072 in its own reasoning, and neither figure comes from a calculation that includes the ~$505/month TANF cash income. On policyengine-us 2.60.0 that income produces net income of $691 and a $208 monthly contribution."
-us,scenario_100,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"It used a $204 standard deduction and a medical deduction the head does not qualify for, and it left out the TANF cash income; taking no utility-allowance shelter deduction matches policyengine-us 2.60.0, and the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua. Its net income was $97, against $691 on policyengine-us 2.60.0, and its $768 maximum allotment is below the FY2026 $785 that the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months."
-us,scenario_100,snap,gemini-4-argon,prompt_ambiguity,age_disability,False,"It brought net income to $0 using a medical deduction the head does not qualify for and ignored the ~$505/month TANF cash income, which leaves positive net income of $691 on policyengine-us 2.60.0. It also used a $768 maximum instead of the FY2026 $785, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months."
-us,scenario_100,snap,glm-5.2,prompt_ambiguity,age_disability,False,"It deducted the $20,644 employer-sponsored insurance premium and other medical costs as a disabled-member medical deduction, but the head is not a SNAP disabled member. It ignored the TANF cash income, so it wrongly set net income to zero. Its $800 allotment for all 12 months is also above the FY2026 $785 maximum, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months."
-us,scenario_100,snap,glm-5.3,prompt_ambiguity,age_disability,False,"It calculated net income of $179.34 from wages alone and missed the ~$505/month TANF cash income; taking no utility-allowance shelter deduction matches policyengine-us 2.60.0, and the frozen reference's $453.64 excess shelter deduction comes from engine defect r30_snap_heat_and_eat_sua. It then applied an $800 allotment to every month instead of the FY2026 $785, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months."
-us,scenario_100,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Its $12,588 (about $1,049/month) exceeds the three-person maximum allotment of $785/month, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months. It also never subtracted 30% of net income, which is $691 on policyengine-us 2.60.0."
-us,scenario_100,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"It set SNAP to zero because no housing costs were listed, but SNAP does not require shelter costs to be eligible. On policyengine-us 2.60.0, which grants this household no utility allowance, the TANF-inclusive net income is $691 and the benefit $577/month; the frozen reference's $453.64 utility-allowance shelter deduction comes from engine defect r30_snap_heat_and_eat_sua."
-us,scenario_100,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"It used the disabled head's insurance premiums and medical costs to bring net income to zero. The head is not a SNAP disabled member (no SSI or SSDI), so no medical deduction applies, and the ~$505/month TANF cash income keeps net income at $691 on policyengine-us 2.60.0. It also used a $767 maximum instead of the FY2026 $785, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months."
-us,scenario_100,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"It gave the full $785 maximum for 12 months on the assumption that countable income is very low. It ignored the ~$505/month TANF cash income, which leaves net income of $691 and a $208 contribution on policyengine-us 2.60.0. Its $785 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026)."
-us,scenario_100,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"It wiped out net income with medical deductions for a disabled member, but the head receives no SSI or other disability benefit and is not a SNAP disabled member. It also ignored the TANF cash income, so it gave the $785 maximum instead of the $577/month ($785 − $208) policyengine-us 2.60.0 gives."
-us,scenario_100,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"It counted SSI income the household does not receive: the head's $2,800 in assets exceeds the $2,000 SSI resource limit. It also applied a medical deduction that policyengine-us 2.60.0 disallows, because the head receives no disability benefit. Its $6,400 is below the $6,924 policyengine-us 2.60.0 gives, which takes no utility-allowance shelter deduction; the frozen reference's $8,625.89 includes a $453.64 shelter deduction from engine defect r30_snap_heat_and_eat_sua and is excluded from scoring."
-us,scenario_100,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"It wiped out net income with insurance and medical deductions, but the head is not a SNAP disabled member because they receive no disability benefit. It also ignored the ~$505/month TANF cash income, which leaves net income of $691 on policyengine-us 2.60.0, so it gave the full $785 for 12 months instead of $577/month."
-us,scenario_100,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"It applied a single $710.92 monthly benefit to all 12 months; holding one schedule all year matches the release's SNAP convention (c_snap_hold_fy2026), and the frozen reference's $733.60 for Oct–Dec 2026 uses the engine's projected FY2027 $802.60 allotment. Its derivation assumed no TANF income. Assuming no shelter costs matches policyengine-us 2.60.0, which with TANF counted gives $577/month; the frozen reference's $453.64 utility-allowance shelter deduction comes from engine defect r30_snap_heat_and_eat_sua."
-us,scenario_100,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"It counted TANF as income and gave no utility-allowance shelter deduction, as policyengine-us 2.60.0 does; the frozen reference's $453.64 excess shelter deduction comes from engine defect r30_snap_heat_and_eat_sua. Its benefit of $618/month is above the $577/month policyengine-us 2.60.0 gives."
-us,scenario_100,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"It counted TANF income and explicitly applied no shelter deduction, as policyengine-us 2.60.0 does; the frozen reference's $453.64 excess shelter deduction comes from engine defect r30_snap_heat_and_eat_sua. It took a medical deduction that policyengine-us 2.60.0 disallows, because the head receives no SSI or other disability benefit. Its $785 × 12 annualization matches the release's SNAP convention (c_snap_hold_fy2026)."
-us,scenario_100,snap,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"It counted TANF and a disability medical deduction and took no utility-allowance shelter deduction, as policyengine-us 2.60.0 does; the frozen reference's $453.64 excess shelter deduction comes from engine defect r30_snap_heat_and_eat_sua. Its $7,764 is above the $6,924 policyengine-us 2.60.0 gives. The medical deduction does not apply because the head receives no SSI or SSDI."
-us,scenario_100,snap,grok-4.3,prompt_ambiguity,age_disability,False,"It returned $0 without calculating anything from the listed inputs. The household qualifies, and policyengine-us 2.60.0 gives net income of $691 and $577/month."
-us,scenario_100,snap,grok-4.5,prompt_ambiguity,age_disability,False,"It used a $198 standard deduction, a $766 maximum allotment and a medical deduction (with the $564 premium counted twice) for a head who is not a SNAP disabled member. It left out the TANF cash income, which produced $56/month of net income against $691 on policyengine-us 2.60.0. Taking no utility-allowance shelter deduction matches that engine; the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua."
-us,scenario_100,snap,grok-4.6,prompt_ambiguity,age_disability,False,"It set net income to $0 through an excess medical deduction the head does not qualify for, and it ignored the ~$505/month TANF cash income. It also used a $766 maximum allotment instead of the FY2026 $785, which the release's SNAP convention (c_snap_hold_fy2026) holds for all 12 months."
-us,scenario_100,snap,grok-4.7,prompt_ambiguity,age_disability,False,"It deducted a $1,688/year excess medical amount, with the $564 premium counted twice, for a head who is not a SNAP disabled member. It omitted the TANF cash income and used a flat $787 allotment ($9,444/year). Taking no utility-allowance shelter deduction matches policyengine-us 2.60.0; the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua."
-us,scenario_100,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,It submitted the $805 monthly allotment as the annual amount instead of summing 12 months. It also set net income to $0 through a medical deduction the head does not qualify for and ignored the TANF income that produces a $208 contribution on policyengine-us 2.60.0.
-us,scenario_100,snap,inkling,prompt_ambiguity,age_disability,False,"It reached net income near $50 with a disabled-member medical deduction the head does not qualify for, and it left out the ~$505/month TANF cash income. On policyengine-us 2.60.0, which takes no utility-allowance shelter deduction, net income is $691, which gives $577/month. The frozen reference's $237.09 and $713.90 include a $453.64 shelter deduction from engine defect r30_snap_heat_and_eat_sua."
-us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no SNAP value and no explanation, so there was no answer to score."
-us,scenario_100,snap,kimi-k3,prompt_ambiguity,age_disability,False,"It set net income to zero with deductions for the disabled head's premiums and medical costs, but the head receives no disability benefit and is not a SNAP disabled member. It also ignored the TANF cash income, so it gave the $785 maximum instead of subtracting the $208 contribution policyengine-us 2.60.0 computes."
-us,scenario_100,snap,minimax-m3,prompt_ambiguity,age_disability,False,"It treated $393 (wages after the 20% earned income deduction, before the standard deduction) as net income. It then cut the benefit to $377/month, which does not follow from $766 − 30% × $393. It also left out the TANF income. Taking no utility-allowance shelter deduction matches policyengine-us 2.60.0; the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua."
-us,scenario_100,snap,muse-spark-1.3,prompt_ambiguity,age_disability,False,"It multiplied a single $710.90 monthly benefit by 12; holding one allotment all year matches the release's SNAP convention (c_snap_hold_fy2026), and the frozen reference's $733.60 for Oct–Dec 2026 uses the engine's projected FY2027 $802.60 allotment. Its net income came from wages less the standard and earned income deductions, without the TANF cash income. Taking no utility-allowance shelter deduction matches policyengine-us 2.60.0; the frozen reference's $453.64 comes from engine defect r30_snap_heat_and_eat_sua."
-us,scenario_100,snap,muse-wonder,prompt_ambiguity,age_disability,False,"It took an excess medical deduction for a disabled member the head does not qualify for, and it left out the ~$505/month TANF cash income. Its net income was $97, against $691 on policyengine-us 2.60.0, which takes no utility-allowance shelter deduction; the frozen reference's $453.64 shelter deduction comes from engine defect r30_snap_heat_and_eat_sua. Its $785 for all 12 months matches the release's SNAP convention (c_snap_hold_fy2026)."
-us,scenario_100,snap,ox-alpha,prompt_ambiguity,age_disability,False,"It deducted $182/month of medical costs, including the second child's $500 and the $564 premium counted twice, as a disabled-member deduction the head does not qualify for. It ignored the TANF cash income, so it got net income of $3, against $691 on policyengine-us 2.60.0."
-us,scenario_100,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"It added the $710 overtime premium on top of wages that already include it. Its $12,444 is also well above its own $766 × 12 = $9,192 and above the legal three-person maximum. On policyengine-us 2.60.0 the benefit is $785 − $208 = $577 for every month."
-us,scenario_100,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"It wrongly concluded that resources exceed SNAP limits. The $2,800 in bank assets is within the resource limit, the vehicle is excluded, and TANF-based categorical eligibility applies, so the household gets $577/month on policyengine-us 2.60.0."
+us,scenario_100,snap,claude-fable-5,prompt_ambiguity,age_disability,False,"Counted only the $493 of monthly earnings and left out about $505 of TANF cash, which is countable unearned income. It also took a $94 excess medical deduction that does not apply, since the head receives no disability benefit, and it missed the $453.64 excess shelter deduction. Its $89 net income (against $237.09) pushed the benefit above $8,625.89."
+us,scenario_100,snap,claude-fable-5.1,prompt_ambiguity,age_disability,False,"Left out the roughly $505/month of TANF income and took a $1,124/year medical deduction for the head, who does not count as SNAP-disabled without SSI or other disability benefits. It also skipped the $453.64/month excess shelter deduction and used a flat $785 for all 12 months. The result was $87/month of net income instead of $237.09."
+us,scenario_100,snap,claude-fable-5.2,prompt_ambiguity,age_disability,False,"Correctly counted TANF, gave no medical deduction, and used the $209 standard deduction and $98.58 earned-income deduction. However, it left out the $453.64 excess shelter deduction, which left net income at $690.65 instead of $237.09. It also ignored the higher allotment for October–December."
+us,scenario_100,snap,claude-haiku-4.5,prompt_ambiguity,age_disability,False,"Never ran the net-income or benefit calculation and picked an arbitrary 'conservative' $1,908. The household is eligible: gross income of $998.31 including TANF, minus the standard, earned-income and shelter deductions, gives $237.09 net income and about $714/month in benefits."
+us,scenario_100,snap,claude-haiku-5.5,prompt_ambiguity,age_disability,False,"Set SNAP to zero with no income test. The household is eligible through TANF-based categorical eligibility, its $2,800 in assets are within the limit, and its $237.09 net income produces about $714/month."
+us,scenario_100,snap,claude-opus-4.7,prompt_ambiguity,age_disability,False,"Built net income from earnings only, leaving out about $505/month of TANF. It took a $141 medical deduction the head does not qualify for, since they receive no disability benefit, and skipped the $453.64 shelter deduction. It then cut its own $9,108 figure down to $7,320 with no stated basis."
+us,scenario_100,snap,claude-opus-4.8,prompt_ambiguity,age_disability,False,"Counted only earnings and left TANF cash out of SNAP income. It used about $200/month as the standard deduction instead of $209 and never applied the $453.64 shelter deduction. It also used a $766 maximum allotment instead of $785, then made an unexplained adjustment to $8,316."
+us,scenario_100,snap,claude-opus-5,prompt_ambiguity,age_disability,False,"Treated net income as earnings less the earned-income and standard deductions only. It left out about $505/month of TANF income and the $453.64 excess shelter deduction, so its rough $8,300 does not follow the engine's $237.09 net income and two-allotment year."
+us,scenario_100,snap,claude-opus-5.5,prompt_ambiguity,age_disability,False,"Left out the roughly $505/month of TANF income and took a $94 medical deduction for the head, who is not SNAP-disabled without SSI or other disability benefits. It also missed the $453.64 shelter deduction, which gave $92 net income instead of $237.09."
+us,scenario_100,snap,claude-sonnet-4.6,prompt_ambiguity,age_disability,False,"Left TANF out of countable income, used a $228 standard deduction instead of $209, and took a $93.67 medical deduction that does not apply to this head. It also treated the shelter deduction as zero instead of $453.64 and used $784 as the maximum for all 12 months."
+us,scenario_100,snap,claude-sonnet-5,prompt_ambiguity,age_disability,False,"Annualized a $766/month maximum as $13,700, which exceeds the yearly maximum for a household of three. It also counted the $20,644 employer insurance premium as a deductible medical expense and left out TANF income. The correct figure is $237.09 net income and about $714/month in benefits."
+us,scenario_100,snap,claude-sonnet-5.5,prompt_ambiguity,age_disability,False,"Reduced net income to $0–$50 with a disabled-member medical deduction that does not apply, because the head receives no disability benefit. It also left out about $505/month of TANF income, so it overstated the benefit relative to the $237.09 net income."
+us,scenario_100,snap,deepseek-v4-flash-0731,prompt_ambiguity,age_disability,False,"Added the $710 overtime premium on top of the $5,915 in wages, even though overtime is already included in gross wages. It also left out TANF income, took a $100.67 medical deduction the head does not qualify for, skipped the $453.64 shelter deduction, and used a $797 maximum allotment."
+us,scenario_100,snap,deepseek-v4-pro,prompt_ambiguity,age_disability,False,"Counted TANF but at $458.54 instead of $505.39, used a $204 standard deduction instead of $209, and added a $46.67 medical deduction that does not apply. It also left out the $453.64 shelter deduction, giving $602 net income instead of $237.09, and used a $768 maximum allotment instead of $785."
+us,scenario_100,snap,deepseek-v4-pro-0813,prompt_ambiguity,age_disability,False,"Counted only about $335/month of TANF (the engine counts $505.39), used a $198 standard deduction, and took a $140.67 medical deduction for a head who is not SNAP-disabled. It skipped the $453.64 shelter deduction and used a $768 maximum. It then submitted $9,037.20, which contradicts the $7,808.48 its own explanation calculated."
+us,scenario_100,snap,deepseek-v4.1-flash,prompt_ambiguity,age_disability,False,"Left TANF out of countable income, used a $204 standard deduction, and took a $53.67 medical deduction that does not apply. It also missed the $453.64 shelter deduction and applied an $805 maximum allotment to all 12 months instead of $785 for January through September."
+us,scenario_100,snap,gemini-3-flash-preview,prompt_ambiguity,age_disability,False,"Took a $140/month medical deduction, counting the $564 premium twice, even though the head receives no disability benefit and so gets no medical deduction. It also left out about $505/month of TANF income and the $453.64 shelter deduction, which gave $54 net income instead of $237.09."
+us,scenario_100,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,age_disability,False,"Gave a near-maximum benefit with no derivation, which implies close to zero net income. That ignores the $505/month of TANF income, which puts net income at $237.09 even after the $453.64 shelter deduction and cuts the benefit by $71.10/month."
+us,scenario_100,snap,gemini-3.1-pro-preview,prompt_ambiguity,age_disability,False,"Called the benefit near-maximum but submitted $6,128 (about $511/month), with no calculation behind it. The correct path counts TANF in gross income ($998.31), subtracts the standard, earned-income and $453.64 shelter deductions, and lands at about $714/month."
+us,scenario_100,snap,gemini-3.5-flash,prompt_ambiguity,age_disability,False,"Correctly counted TANF as unearned income but took a medical deduction the head does not qualify for, since they receive no disability benefit. It also left out the $453.64 excess shelter deduction and used a $768 maximum allotment instead of $785, which kept the benefit near $605/month."
+us,scenario_100,snap,gemini-3.5-flash-lite,prompt_ambiguity,age_disability,False,"Wrongly concluded that income and resource limits ruled the household out. Gross income is 45% of the poverty guideline, the $2,800 in assets is within the limit, and TANF-based categorical eligibility applies."
+us,scenario_100,snap,gemini-3.6-flash,prompt_ambiguity,age_disability,False,"Started from $492.92 of wages alone, leaving out about $505/month of TANF. It applied a disabled-head medical deduction that does not exist here because the head receives no disability benefit, and it skipped the $453.64 shelter deduction."
+us,scenario_100,snap,gemini-3.7-flash,prompt_ambiguity,age_disability,False,"Submitted $9,216 ($768 × 12), which implies zero net income, while its explanation says $9,072. It ignored the TANF income that leaves $237.09 net, which cuts $71.10/month from a $785 maximum."
+us,scenario_100,snap,gemini-3.8-flash,prompt_ambiguity,age_disability,False,"Counted only wages, leaving out TANF, and used a $204 standard deduction plus a medical deduction that does not apply to this head. It also missed the $453.64 shelter deduction and used a $768 maximum instead of $785."
+us,scenario_100,snap,gemini-4-argon,prompt_ambiguity,age_disability,False,Reduced net income to $0 with a medical deduction for a head who receives no disability benefit and so gets none. It also ignored about $505/month of TANF and used a $768 maximum instead of $785.
+us,scenario_100,snap,glm-5.2,prompt_ambiguity,age_disability,False,"Counted the $20,644 employer-sponsored insurance premium and other expenses as a medical deduction to wipe out net income, but the head is not SNAP-disabled and gets no medical deduction. It also left out TANF income and used an $800/month maximum."
+us,scenario_100,snap,glm-5.3,prompt_ambiguity,age_disability,False,"Calculated net income from earnings only ($179), leaving out the $505/month of TANF income and the $453.64 shelter deduction, which together give $237.09. It also applied an $800 maximum to all 12 months instead of $785 for January through September."
+us,scenario_100,snap,gpt-5.4-mini,prompt_ambiguity,age_disability,False,"Used a monthly maximum of about $1,049, which is above the $785 maximum for a household of three. It also subtracted nothing for countable income, although $237.09 of net income cuts $71.10/month from the benefit."
+us,scenario_100,snap,gpt-5.4-nano,prompt_ambiguity,age_disability,False,"Treated missing rent and utility details as making SNAP zero, but SNAP does not require shelter costs. The household is categorically eligible and gets about $714/month from $237.09 of net income."
+us,scenario_100,snap,gpt-5.5,prompt_ambiguity,age_disability,False,"Reduced net income to zero by deducting the head's health insurance and medical costs, but the head receives no disability benefit, so no medical deduction applies. It also ignored about $505/month of TANF income and used a $767 maximum."
+us,scenario_100,snap,gpt-5.6-luna,prompt_ambiguity,age_disability,False,"Paid the full $785 maximum for 12 months, which assumes zero net income. TANF plus earnings give $998.31 gross and $237.09 net, so $71.10/month comes off. It also ignored the higher October–December allotment."
+us,scenario_100,snap,gpt-5.6-sol,prompt_ambiguity,age_disability,False,"Used a disabled-member medical deduction to reach zero net income, but the head receives no SSI or other disability benefit and gets no medical deduction. It also left out the TANF income that leaves $237.09 net and a $71.10/month reduction."
+us,scenario_100,snap,gpt-5.6-terra,prompt_ambiguity,age_disability,False,"Counted SSI income, although the head gets no SSI because their $2,800 in assets exceeds the SSI resource limit. It also applied a medical deduction, did not use the $505.39 of TANF income, and left out the $453.64 shelter deduction, leaving about $533/month instead of about $714."
+us,scenario_100,snap,gpt-6-astra,prompt_ambiguity,age_disability,False,"Deducted the head's insurance and medical expenses to reach zero net income, but the head is not SNAP-disabled, so no medical deduction applies. It also ignored about $505/month of TANF income, which makes net income $237.09."
+us,scenario_100,snap,gpt-6-luna,prompt_ambiguity,age_disability,False,"Explicitly assumed no shelter deduction and calculated from earnings alone. It missed both the $505.39 of TANF income and the $453.64 excess shelter deduction, and paid one flat $710.92 for 12 months instead of $713.90 for January–September and $733.60 for October–December."
+us,scenario_100,snap,gpt-6-sol,prompt_ambiguity,age_disability,False,"Counted estimated TANF in income but applied only the earned-income and standard deductions. It left out the $453.64 excess shelter deduction, so net income stayed too high and the benefit came to $618/month instead of about $714."
+us,scenario_100,snap,gpt-6.1-sol,prompt_ambiguity,age_disability,False,"Explicitly applied no shelter deduction, where the engine subtracts $453.64/month. It also took a disabled-head medical deduction, although the head receives no disability benefit and gets none."
+us,scenario_100,snap,gpt-6.1-terra,prompt_ambiguity,age_disability,False,"Counted TANF but replaced the $453.64 excess shelter deduction with a disability-related medical deduction the head does not qualify for. Net income ended up well above $237.09, and the benefit came to $7,764."
+us,scenario_100,snap,grok-4.3,prompt_ambiguity,age_disability,False,"Returned zero without doing any SNAP calculation. The household is categorically eligible, has gross income of $998.31 and net income of $237.09, and gets about $714/month."
+us,scenario_100,snap,grok-4.5,prompt_ambiguity,age_disability,False,"Left out TANF income, used a $198 standard deduction instead of $209, and took a $1,688/year medical deduction, counting the $564 premium twice, for a head who is not SNAP-disabled. It also missed the $453.64 shelter deduction and used a $766 maximum."
+us,scenario_100,snap,grok-4.6,prompt_ambiguity,age_disability,False,"Used an excess medical deduction to bring net income to $0, but no medical deduction applies because the head receives no disability benefit. It also left out about $505/month of TANF and used a $766 maximum instead of $785."
+us,scenario_100,snap,grok-4.7,prompt_ambiguity,age_disability,False,"Correctly kept the $20,644 employer premium out, but still took a $1,688/year medical deduction for a head who is not SNAP-disabled. It also left TANF out of income and never applied the $453.64/month shelter deduction."
+us,scenario_100,snap,grok-build-0.1,prompt_ambiguity,age_disability,False,"Reported a single monthly allotment ($805) instead of the annual total. It also reduced net income to $0 with a medical deduction that does not apply, and ignored the TANF income that leaves $237.09 net."
+us,scenario_100,snap,inkling,prompt_ambiguity,age_disability,False,"Counted only wages, leaving out about $505/month of TANF, and applied a disabled-member medical deduction the head does not qualify for. It also missed the $453.64 shelter deduction, which pushed net income down to about $50 instead of $237.09."
+us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there was no answer to score."
+us,scenario_100,snap,kimi-k3,prompt_ambiguity,age_disability,False,"Reduced net income to zero with the head's premiums and medical expenses, but the head receives no disability benefit and gets no medical deduction. It also ignored about $505/month of TANF income, which leaves $237.09 net and a $71.10/month reduction."
+us,scenario_100,snap,minimax-m3,prompt_ambiguity,age_disability,False,"Subtracted all of the roughly $393 net income from the maximum allotment instead of 30% of it. It also never applied the standard deduction or the $453.64 shelter deduction, and it left TANF out of income."
+us,scenario_100,snap,muse-spark-1.3,prompt_ambiguity,age_disability,False,"Calculated net income from earnings with only the standard and earned-income deductions. It left out the $505/month of TANF and the $453.64 shelter deduction, and used a $768 maximum instead of $785 for January–September and the higher allotment for October–December."
+us,scenario_100,snap,muse-wonder,prompt_ambiguity,age_disability,False,"Used a disabled-member excess medical deduction to bring net income down to $97, but the head is not SNAP-disabled and gets no medical deduction. It also left out about $505/month of TANF income and the $453.64 shelter deduction."
+us,scenario_100,snap,ox-alpha,prompt_ambiguity,age_disability,False,"Took a $182/month medical deduction based on a double-counted $2,608 in out-of-pocket costs, for a head who is not SNAP-disabled. That brought net income to $3 instead of $237.09, and it also ignored TANF income and the shelter deduction."
+us,scenario_100,snap,qwen-3.7-max,prompt_ambiguity,age_disability,False,"Added the $710 overtime premium on top of wages that already include it, and turned $766 × 12 into $12,444, which exceeds the yearly maximum for three people ($766 × 12 is $9,192). It also left TANF out of countable income."
+us,scenario_100,snap,qwen3.8-max,prompt_ambiguity,age_disability,False,"Wrongly concluded that resources exceed the SNAP limit. The $2,800 bank balance is within the limit, the vehicle is excluded, and TANF-based categorical eligibility applies, so the household gets about $714/month."
us,scenario_100,ssi,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the head's generic disability fact as satisfying SSI's categorical disability requirement and proceeded directly to the earned-income calculation. PolicyEngine's SSI-specific aged/blind/disabled indicator is false for the head, so no SSI benefit-rate or income-exclusion calculation applies; its submitted value also contradicts its own stated $9,480 calculation."
us,scenario_100,ssi,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model equated the head's generic disability status with SSI categorical eligibility. The head's SSI-specific aged/blind/disabled indicator is false, so the earned-income reduction it applied is irrelevant and SSI is zero."
us,scenario_100,ssi,gpt-5.5,llm_error,categorical_eligibility,False,"The model assumed the head was an SSI-qualified disabled adult and subtracted countable earnings from a federal benefit maximum. The head fails SSI's aged, blind, or qualifying-disabled criterion, so the calculation stops before income exclusions or the federal benefit rate are applied."
@@ -8831,149 +8927,152 @@ us,scenario_100,ssi,grok-build-0.1,llm_error,categorical_eligibility,False,"The
us,scenario_100,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract."
us,scenario_100,ssi,ox-alpha,llm_error,categorical_eligibility,False,"The model assumed the 46-year-old head's generic disability status met SSI's qualifying-disability test and then calculated countable wages. The SSI-specific categorical indicator is false, so the head is ineligible regardless of the wage calculation; neither child qualifies either."
us,scenario_100,ssi,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated the generic disability input as SSI categorical eligibility and therefore performed an income-reduction calculation that should never occur. It also added the separately listed overtime premium to annual gross wages and invented an IRWE adjustment despite instructions not to infer unlisted expenses, but categorical ineligibility alone makes SSI zero."
-us,scenario_100,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used 10% of the federal EITC instead of Montana's 20% rate for 2026. It also went back and forth on earned income, landing on an incorrect $2,646 figure instead of $2,055.72 (wages minus the $3,859 of 401(k) deferrals). Its final $265 does not follow from any of its own calculations; the correct result is 20% × $822.29 = $164.46."
-us,scenario_100,state_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It got the federal EITC about right (~$822) by netting out the 401(k) deferrals, but applied the outdated 10% Montana rate instead of the 20% rate in effect for 2026. That halved the credit to $82.24 instead of $164.46."
-us,scenario_100,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It claimed Montana has no refundable earned income credit and pointed to the filing threshold. Montana's EITC is refundable, worth 20% of the federal EITC, and is paid even with no state tax liability, so this filer is owed $164.46."
-us,scenario_100,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It gave no reasoning and answered $0, which means it missed Montana's refundable EITC. The correct amount is 20% of the $822.29 federal EITC, or $164.46."
-us,scenario_100,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It figured the federal EITC on gross wages ($2,366) without taking out the $3,859 of traditional 401(k) deferrals, and it used a 10% Montana rate instead of 20%. Its own math gave $237, but it then submitted $297 with no basis. The correct figure is 20% × $822.29 = $164.46."
-us,scenario_100,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It correctly found that the federal EITC is positive and that Montana's EITC is a share of it, then zeroed the credit anyway as a 'conservative estimate'. Montana's EITC is refundable, so the household receives 20% × $822.29 = $164.46."
-us,scenario_100,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It treated the Montana EITC as nonrefundable and let zero tax liability wipe it out. The credit is refundable at 20% of the federal EITC, giving $164.46."
-us,scenario_100,state_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It computed the federal EITC correctly (~$822) on 401(k)-reduced earned income but used Montana's old 10% rate instead of the 20% rate for 2026. The result, $82.24, is half of the $164.46 reference."
-us,scenario_100,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It ran the federal EITC on gross wages of $5,915 ($2,366), even after noting that the 401(k) deferrals bring AGI down to $2,056. EITC earned income excludes those deferrals. It also used 10% instead of Montana's 20% rate, giving $237 instead of 20% × $822.29 = $164.46."
-us,scenario_100,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It wrongly called the Montana EITC nonrefundable and set refundable credits to $0. The credit is refundable at 20% of the $822.29 federal EITC, giving $164.46."
-us,scenario_100,state_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It took 10% of a $2,366 federal EITC. That makes two mistakes: Montana's rate is 20%, and the federal EITC should be 40% of $2,055.72 (wages minus 401(k) deferrals) = $822.29. The correct credit is $164.46."
-us,scenario_100,state_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a 12% Montana rate, which does not exist, and an inflated $2,650 federal EITC. The correct inputs are a 20% rate and a federal EITC of 40% × $2,055.72 = $822.29, which gives $164.46."
-us,scenario_100,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a 10% Montana rate and an unexplained $371 federal EITC. The federal EITC is 40% × $2,055.72 = $822.29 and Montana pays 20% of it, giving $164.46."
-us,scenario_100,state_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied the old 3% Montana EITC rate to an otherwise correct ~$822 federal EITC, instead of the 20% rate for 2026. It also added a $1,200 young-child credit that is not among this household's refundable Montana credits. The correct total is the EITC alone: $164.46."
-us,scenario_100,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,It said Montana has no refundable credits for this household. That misses the refundable Montana EITC of 20% × $822.29 = $164.46.
-us,scenario_100,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It took 10% of a $2,366 federal EITC that was computed on gross wages. Montana's rate is 20%, and EITC earned income excludes the $3,859 of 401(k) deferrals, so the federal credit is $822.29 and Montana's is $164.46."
-us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It found no Montana refundable credits and missed the refundable EITC, which is 20% of the $822.29 federal EITC, or $164.46."
-us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It claimed Montana has no refundable EITC and treated earned income as zero. The head has $2,055.72 of earned income after 401(k) deferrals, which produces an $822.29 federal EITC and a refundable Montana EITC of 20% of that, $164.46."
-us,scenario_100,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"It treated the federal EITC as $0 and zeroed the Montana EITC as a result. A head of household with two qualifying children and $2,055.72 of earned income is in the 40% phase-in range and gets $822.29, so Montana's 20% credit is $164.46."
-us,scenario_100,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It gave no reasoning and answered $0, which ignores Montana's refundable EITC of 20% × $822.29 = $164.46."
-us,scenario_100,state_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used the correct 20% Montana rate but based the federal EITC on gross wages ($2,366). EITC earned income excludes the $3,859 of traditional 401(k) deferrals, leaving $2,055.72 and a federal EITC of $822.29. The correct result is $164.46, not $473.20."
-us,scenario_100,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its reasoning used the old 3% rate on a gross-wage federal EITC of $2,366 to get $70.98, yet it submitted $0. The correct figure is 20% of the $822.29 federal EITC computed on 401(k)-reduced earned income, or $164.46."
-us,scenario_100,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,It wrongly said Montana refundable credits are not modeled. The engine computes a refundable Montana EITC of 20% × $822.29 = $164.46.
-us,scenario_100,state_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"It used Montana's old 10% rate instead of 20%, and a federal EITC of $2,366 computed on gross wages instead of $822.29 on wages minus 401(k) deferrals. That gave $236.60 instead of $164.46."
-us,scenario_100,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It wrongly claimed Montana has no individual income tax. Montana does have one, with a refundable EITC equal to 20% of the federal EITC, which is $164.46 here."
-us,scenario_100,state_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used Montana's old 3% EITC rate instead of 20%, applied to a gross-wage federal EITC of $2,366 instead of $822.29 on 401(k)-reduced earned income. The correct result is $164.46."
-us,scenario_100,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It saw no Montana refundable credits and missed the refundable state EITC, which is 20% of the $822.29 federal EITC, or $164.46."
-us,scenario_100,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,It returned $0 without identifying Montana's refundable EITC. The credit is 20% × $822.29 = $164.46 for this low-income head of household with two children.
-us,scenario_100,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It inflated the federal EITC to $4,207 even though earned income is only $2,055.72 at a 40% phase-in rate ($822.29), and it used 10% instead of Montana's 20% rate. The correct figure is $164.46."
-us,scenario_100,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It took 10% of a $2,366 federal EITC computed on gross wages. Montana's 2026 rate is 20%, and earned income must exclude the $3,859 of 401(k) deferrals, which gives 20% × $822.29 = $164.46."
-us,scenario_100,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"It added a $1,200 young-child credit for the 5-year-old, but that credit is not among this household's refundable Montana credits. It also computed the EITC as 10% of a gross-wage $2,366 federal credit instead of 20% of $822.29. The correct total is $164.46."
-us,scenario_100,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It included a $1,200 young-child credit, which adds nothing to this household's refundable Montana credits. It also computed the EITC at 10% of a $2,366 federal EITC based on gross wages instead of 20% of $822.29. The correct total is $164.46."
-us,scenario_100,state_refundable_credits,gpt-6-luna,llm_error,categorical_eligibility,False,"It invented $1,800 of Montana child tax credit for the 5- and 6-year-olds; those credits are not part of the refundable total. It also ran the 20% EITC on a gross-wage $2,366 federal EITC instead of the $822.29 based on wages net of 401(k) deferrals. The correct answer is $164.46."
-us,scenario_100,state_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It used the correct 20% Montana rate but a federal EITC of $2,366 computed on gross wages. Traditional 401(k) deferrals are excluded from EITC earned income, leaving $2,055.72 and a federal EITC of $822.29, so the answer is $164.46."
-us,scenario_100,state_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It applied a 10% rate to a gross-wage federal EITC of $2,366. Montana pays 20% of a federal EITC of $822.29, which is computed on wages minus the $3,859 of 401(k) deferrals, giving $164.46."
-us,scenario_100,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,It claimed no Montana refundable credits apply and missed the refundable EITC of 20% × $822.29 = $164.46.
-us,scenario_100,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"It wrongly stated that Montana has no state EITC. Montana has a refundable EITC equal to 20% of the federal EITC, which yields $164.46 for this household."
-us,scenario_100,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,It wrongly asserted that Montana has no state EITC and only looked at property-tax credits. The refundable Montana EITC at 20% of the $822.29 federal credit is $164.46.
-us,scenario_100,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It ruled out Montana credits using the age-62 and rent tests for the elderly homeowner/renter credit, and wrongly claimed there is no state EITC. Montana's refundable EITC of 20% × $822.29 = $164.46 applies."
-us,scenario_100,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It returned $0 and missed Montana's refundable EITC, which is 20% of the $822.29 federal EITC, or $164.46."
-us,scenario_100,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"It added a $1,200 refundable child credit that is not part of this household's Montana refundable credits, and it used a 10% EITC rate on a ~$750 federal EITC. The correct answer is the EITC alone: 20% × $822.29 = $164.46."
-us,scenario_100,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"It wrongly stated that Montana has no state EITC and only checked the circuit-breaker credit. Montana's refundable EITC equals 20% of the $822.29 federal EITC, or $164.46."
-us,scenario_100,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It used 10% instead of the 20% rate for 2026, applied to a $2,366 federal EITC based on gross wages instead of $822.29 on 401(k)-reduced earned income. The correct result is $164.46."
-us,scenario_100,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It wrongly said Montana has no state income tax. Montana taxes income and pays a refundable EITC of 20% of the federal EITC, which is $164.46 here."
-us,scenario_100,state_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It took 10% of a $2,366 gross-wage federal EITC. The correct inputs are Montana's 20% rate and a federal EITC of 40% × $2,055.72 = $822.29, which gives $164.46."
-us,scenario_100,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"It wrongly treated Montana's earned income credit as nonrefundable. The credit is refundable at 20% of the federal EITC, so it pays $164.46 despite zero liability."
-us,scenario_100,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It applied 10% instead of Montana's 20% rate, to a federal EITC of $2,366 computed on gross wages instead of $822.29 on wages minus 401(k) deferrals. The correct result is $164.46."
-us,scenario_100,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It wrongly asserted that Montana has no earned income tax credit. Montana's refundable EITC is 20% of the $822.29 federal EITC, or $164.46."
-us,scenario_100,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,It said no Montana refundable credits apply and missed the refundable EITC of 20% × $822.29 = $164.46.
-us,scenario_100,tanf,claude-fable-5,llm_error,thresholds_rates,False,"It correctly found countable income of about $220/month, below the payment standard, and a positive grant. It then reversed itself and returned $0, claiming the earnings exceed the standards after disregards. Subtracting $219.69 from the roughly $725 three-person standard gives about $505/month, or $6,065/year."
-us,scenario_100,tanf,claude-fable-5.1,llm_error,categorical_eligibility,False,"It assumed Montana TANF is not modeled and returned $0. The program is modeled and pays about $725 minus $219.69 in countable earnings, roughly $505/month, or $6,065/year."
-us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"It counted 4 people instead of 3 and claimed $5,915 of wages exceeds the income limit, when countable income after the $200 + 25% disregard is only $219.69/month. It also applied a $2,000 asset limit, but Montana's limit is $3,000, so the $2,800 balance passes."
-us,scenario_100,tanf,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It returned $0 because no TANF receipt was listed, ignoring the instruction to assume take-up. The eligible 3-person unit's computed grant is about $505/month ($6,065/year)."
-us,scenario_100,tanf,claude-opus-4.7,llm_error,thresholds_rates,False,"It defaulted to $0 citing work requirements and asset limits, but the disabled head is exempt from the work requirement and the $2,800 balance is under the $3,000 limit with the vehicle excluded. Countable earnings of $219.69 against the roughly $725 standard leave about $505/month."
-us,scenario_100,tanf,claude-opus-4.8,llm_error,thresholds_rates,False,"It assumed a $588 maximum, then gave a $595/month benefit that exceeds that maximum, so it never actually subtracted countable earnings. The correct result is the roughly $725 standard minus $219.69 of countable earnings, which is $505.41/month ($6,064.96/year), not $7,140."
-us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"It asserted that PolicyEngine gives this household no Montana TANF and did no calculation. The unit passes the income and $3,000 resource tests and receives about $725 − $219.69 = $505.41/month."
-us,scenario_100,tanf,claude-opus-5.5,llm_error,thresholds_rates,False,"The method was right ($200 + 25% disregard leaves about $220 countable, subtracted from the standard), but it used a $730 payment standard instead of about $725.10 and rounded the result to $510/month. That gives $6,120 instead of $505.41 × 12 = $6,064.96."
-us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"It applied a $1,000 asset limit and ruled the family ineligible because of the $2,800 balance. Montana's TANF resource limit is $3,000, so the household passes and receives about $505/month."
-us,scenario_100,tanf,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $327/month maximum grant, far below the roughly $725 three-person standard, and paid the near-maximum without subtracting the $219.69 of countable earnings. The correct benefit is about $505/month."
-us,scenario_100,tanf,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It assumed Montana TANF is not modeled and returned $0. The engine computes the roughly $725 standard minus $219.69 of countable earnings, about $6,065/year."
-us,scenario_100,tanf,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It added the $1,981 vehicle to the $2,800 bank balance when testing resources. Montana excludes the vehicle, so countable resources of $2,800 are under the $3,000 limit and the family receives about $505/month."
-us,scenario_100,tanf,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $605 maximum grant and a $200 + 50% disregard. Montana's disregard is $200 + 25%, which leaves $219.69 countable, and the three-person standard is about $725, so the benefit is $505.41/month, not $458.54."
-us,scenario_100,tanf,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It submitted $0, contradicting its own explanation's $4,023.75. That figure was also wrong because it used a $555 payment standard instead of about $725. With the correct $219.69 of countable earnings, the benefit is $505.41/month."
-us,scenario_100,tanf,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $500 maximum and a 50% disregard. The correct rule is a $200 + 25% disregard, leaving $219.69 countable, against a roughly $725 three-person standard, for $505.41/month."
-us,scenario_100,tanf,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the disregard correctly (about $220 countable) but used an outdated $588 payment standard instead of about $725, which understates the benefit by about $137/month. It also arbitrarily changed $4,416 to $4,419."
-us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It declared the household ineligible without naming any failed test. The 3-person unit passes Montana's income test ($219.69 countable vs. a roughly $725 standard) and the $3,000 resource test."
-us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It assumed the head receives SSI, but the $2,800 bank balance exceeds SSI's $2,000 individual resource limit, so SSI is $0. With no SSI, TANF is about $725 − $219.69 = $505/month."
-us,scenario_100,tanf,gemini-3.5-flash,llm_error,thresholds_rates,False,"Its $5,298 equals a $588 standard minus earnings after a $200 + 50% disregard ($146.46). Montana disregards $200 + 25% ($219.69 countable), and the three-person standard is about $725."
-us,scenario_100,tanf,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It returned $0 with no reasoning. The eligible 3-person unit receives about $725 − $219.69 = $505.41/month, or $6,064.96/year."
-us,scenario_100,tanf,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $5,298.48 is exactly ($588 − (0.5 × $292.92)) × 12, so it used an outdated $588 standard and a 50% disregard. The correct rule is $200 + 25% against a roughly $725 standard."
-us,scenario_100,tanf,gemini-3.7-flash,llm_error,thresholds_rates,False,"It submitted $0 even though its explanation said the family qualifies and gave $4,824 ($402/month). That figure also understates the roughly $725 standard minus $219.69 countable earnings, which is $505.41/month."
-us,scenario_100,tanf,gemini-3.8-flash,llm_error,categorical_eligibility,False,It asserted that Montana TANF is not modeled. The engine models it and pays about $505/month after subtracting $219.69 of countable earnings from the roughly $725 standard.
-us,scenario_100,tanf,gemini-4-argon,llm_error,thresholds_rates,False,"Its $6,132 implies a $511/month grant, so it overstated the net of the payment standard (about $725.10) minus countable earnings ($219.69). The correct grant is $505.41/month."
-us,scenario_100,tanf,glm-5.2,llm_error,taxable_income_or_deductions,False,"It added the $710 overtime premium to wages that already include overtime, and compared income to the standard without applying the $200 + 25% earned-income disregard. Countable income is only $219.69/month, well under the roughly $725 standard."
-us,scenario_100,tanf,glm-5.3,llm_error,categorical_eligibility,False,It assumed PolicyEngine gives no Montana TANF benefit. The engine pays about $505/month: the roughly $725 three-person standard minus $219.69 of countable earnings.
-us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It returned $0 because TANF payment was not indicated, ignoring the assumed take-up. The 3-person unit qualifies and receives about $6,065/year."
-us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It required an explicit eligibility trigger. Two children under 18 with a parent plus low countable earnings ($219.69/month) are sufficient, producing about $505/month."
-us,scenario_100,tanf,gpt-5.5,llm_error,thresholds_rates,False,It understated countable earnings at $150 (the $200 + 25% disregard leaves $219.69) and implicitly used a $588 standard instead of about $725. The correct grant is $505.41/month.
-us,scenario_100,tanf,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $359.50/month grant reflects an outdated payment standard near $588. The 2026 three-person standard is about $725, which minus $219.69 of countable earnings gives $505.41/month."
-us,scenario_100,tanf,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $4,419.75 equals ($588 − $219.69) × 12. The disregard was right, but $588 is an outdated three-person payment standard; the 2026 standard is about $725."
-us,scenario_100,tanf,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It counted SSI as income, but the head's $2,800 bank balance exceeds SSI's $2,000 resource limit, so there is no SSI. Only $219.69 of countable earnings offsets the roughly $725 standard."
-us,scenario_100,tanf,gpt-6-astra,llm_error,thresholds_rates,False,"The method was right ($200 + 25% disregard, $3,000 resource limit, vehicle excluded), but it rounded the 2026 three-person payment standard to $725. The engine's standard is about $725.10, which gives $6,064.96 instead of $6,063.75."
-us,scenario_100,tanf,gpt-6-luna,llm_error,thresholds_rates,False,"Its $5,298 equals a $588 standard minus a $200 + 50% disregarded amount. Montana uses $200 + 25% ($219.69 countable) against a roughly $725 standard."
-us,scenario_100,tanf,gpt-6-sol,llm_error,thresholds_rates,False,"Its $368.25/month implies an outdated $588 payment standard. The 2026 three-person standard is about $725, giving $505.41/month after the $219.69 of countable earnings."
-us,scenario_100,tanf,gpt-6.1-sol,llm_error,thresholds_rates,False,"It explicitly used a $588 three-person payment standard. The engine's 2026 standard is about $725, so the correct grant is $505.41/month, not $368.31."
-us,scenario_100,tanf,gpt-6.1-terra,llm_error,thresholds_rates,False,"Its $4,419.75 equals ($588 − $219.69) × 12, using an outdated $588 standard instead of the 2026 standard of about $725."
-us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,It did not calculate TANF and returned $0. The eligible 3-person unit receives about $725 − $219.69 = $505.41/month.
-us,scenario_100,tanf,grok-4.5,llm_error,thresholds_rates,False,"It applied the $200 + 25% disregard correctly but used an outdated $588 payment standard instead of the 2026 standard of about $725, understating the grant by about $137/month."
-us,scenario_100,tanf,grok-4.6,llm_error,thresholds_rates,False,"It used a $588 three-person payment standard. The 2026 standard is about $725, so the grant is $505.41/month rather than $368."
-us,scenario_100,tanf,grok-4.7,llm_error,thresholds_rates,False,"Its countable earnings ($219.69) and resource test were correct, but it used an outdated $588 payment standard instead of the 2026 three-person standard of about $725.10."
-us,scenario_100,tanf,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $675 maximum, below the 2026 three-person standard of about $725, giving $455/month instead of $505.41."
-us,scenario_100,tanf,inkling,llm_error,thresholds_rates,False,"It used a $588 maximum grant. The 2026 three-person standard is about $725, so after $219.69 of countable earnings the benefit is $505.41/month, not $368."
-us,scenario_100,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no TANF value or explanation. The correct derivation gives about $725 − $219.69 = $505.41/month, or $6,064.96/year."
-us,scenario_100,tanf,kimi-k3,llm_error,thresholds_rates,False,"Its resource test and $219.69 of countable earnings were correct, but it used a $588 three-person grant instead of the 2026 standard of about $725.10."
-us,scenario_100,tanf,minimax-m3,llm_error,thresholds_rates,False,"It assumed Montana benefits are only $50–$200/month and wiped them out with earnings. The three-person standard is about $725, and after the $200 + 25% disregard only $219.69 is countable."
-us,scenario_100,tanf,muse-spark-1.3,llm_error,thresholds_rates,False,Its $434/month grant implies a payment standard (or disregard) that is too low. The roughly $725 standard minus $219.69 of countable earnings gives $505.41/month.
-us,scenario_100,tanf,muse-wonder,llm_error,thresholds_rates,False,"Its $368/month implies an outdated $588 payment standard. The 2026 three-person standard is about $725, giving $505.41/month."
-us,scenario_100,tanf,ox-alpha,llm_error,categorical_eligibility,False,"It counted about $780/month of SSI as unearned income, but the head's $2,800 bank balance exceeds SSI's $2,000 resource limit, so SSI is $0. Without it, $219.69 of countable earnings against the roughly $725 standard leaves $505.41/month."
-us,scenario_100,tanf,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $710 overtime premium to wages that already include it, and compared gross annual earnings to an annual limit without applying the $200 + 25% disregard. Countable income is $219.69/month against a roughly $725 standard."
-us,scenario_100,tanf,qwen3.8-max,llm_error,thresholds_rates,False,"It claimed income is too high, but $492.92/month of earnings drops to $219.69 countable after Montana's $200 + 25% disregard, well below the roughly $725 standard. The family receives $505.41/month."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It deducted $1,400 of auto loan interest as qualified vehicle-loan interest, but no fact establishes a qualifying new, U.S.-assembled vehicle loan. It then swapped in 2025-style bracket thresholds ($12,525/$50,900), and its final $12,064 matches none of its own computations. The correct result is taxable income of $85,030 taxed at 2026 brackets of $12,400/$50,400, giving $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It subtracted a $1,000 qualified auto-loan-interest deduction (after phaseout) on top of the $16,100 standard deduction, lowering taxable income to $84,030. The household's auto loan interest is not qualified passenger-vehicle loan interest under the OBBBA deduction, so taxable income stays at $85,030 and tax is $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It wrongly added the $700 of nontaxable veterans benefits to AGI and used the 2024 standard deduction of $14,600 instead of the 2026 amount of $16,100. It also taxed everything above $11,600 at 12%, skipping the 22% bracket, and then reported $11,435, which does not match its own figure."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It subtracted health premiums from wages as pre-tax, but the listed gross wages carry no such pre-tax exclusion. Its rough 'approximate 2026 brackets' estimate of $13,900 overstates the correct tax of 1,240 + 4,560 + 22% × 34,630 = $13,418.60 on $85,030 of taxable income."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its own reasoning computed exactly $13,418.60 from $85,030 of taxable income at the 2026 brackets. It then threw that result away and reported an unexplained $14,716 as 'rounding to bracket estimates'."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It correctly derived taxable income of $85,030 and tax of about $13,419 under the 2026 brackets. It then replaced that figure with an unsupported $13,876, citing 'bracket thresholds' it never stated."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It deducted the full $1,400 of auto loan interest as OBBBA qualified vehicle-loan interest, cutting taxable income to $83,630. The facts never establish a qualifying new, U.S.-assembled vehicle loan, so no such deduction applies. Taxable income is $85,030 and tax is $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used the 2025 OBBBA standard deduction of $15,750 instead of the 2026 amount of $16,100. It also used guessed brackets ($12,300/$49,950) instead of the actual 2026 thresholds of $12,400/$50,400. The result is taxable income of $85,380 and tax of $13,543 rather than $85,030 and $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but used wrong bracket thresholds ($12,150/$49,400) instead of $12,400/$50,400, getting $13,523.60. It then reported an unexplained $13,322."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It took a $1,174 qualified auto-loan-interest deduction, lowering taxable income to $83,856. The household's auto loan interest is not qualified vehicle-loan interest under the OBBBA provision, so taxable income is $85,030 and tax is $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,600 standard deduction instead of the 2026 single amount of $16,100. It also used bracket thresholds of $12,175/$49,550 instead of $12,400/$50,400, which inflated taxable income to $85,530 and tax to $13,618."
-us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It counted the same $1,920 premium twice and subtracted $3,840 as pre-tax health premiums, although the listed gross wages carry no pre-tax exclusion. It also used a $15,400 standard deduction instead of $16,100 and wrong bracket thresholds. The correct figures are taxable income of $85,030 and tax of $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and applied a pre-TCJA $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. OBBBA made the TCJA structure permanent, so 2026 uses the $16,100 standard deduction, no exemption, and 10/12/22% brackets, giving $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $15,350 standard deduction instead of the 2026 amount of $16,100. It also used bracket thresholds of $12,200/$49,575 instead of $12,400/$50,400, producing $85,780 of taxable income and $13,670.10 of tax."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunset and applied an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets, even though OBBBA kept the TCJA structure with a $16,100 standard deduction for 2026. It also subtracted the $1,920 premium from wages as pre-tax, which the facts do not support."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It left out the $1,870 capital loss, which is deductible against ordinary income under the $3,000 limit. It also used a $15,300 standard deduction instead of $16,100, so it taxed $87,700 instead of $85,030."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It got AGI right at $101,130 but never applied the 2026 $16,100 standard deduction and the $12,400/$50,400 bracket thresholds. Its unsupported $15,400 overstates the correct $13,418.60 by about $2,000, which is consistent with too small a deduction or a higher rate schedule."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied 'post-TCJA expiration' rules: an $8,250 standard deduction, a $5,300 personal exemption, and 10/15/25% brackets. OBBBA made the TCJA rates permanent, so 2026 uses the $16,100 standard deduction and 10/12/22% brackets, giving $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It never carried out a bracket calculation and reported $11,102. Correctly, $85,030 of taxable income (AGI $101,130 minus the $16,100 standard deduction) taxed at the 2026 brackets gives $13,418.60, so its figure understates tax by about $2,300."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed the TCJA provisions expire in 2026 and used an $8,300 standard deduction, a $5,300 exemption, and 10/15/25% brackets. OBBBA kept the TCJA rates with a $16,100 standard deduction, which gives $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It applied a pre-TCJA $8,300 standard deduction and a $5,300 personal exemption on the assumption that TCJA sunsets. OBBBA made the $16,100 standard deduction and 10/12/22% brackets permanent for 2026, so taxable income is $85,030 and tax is $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 and the 2025 bracket thresholds ($11,925/$48,475). It should have used the 2026 values of $16,100 and $12,400/$50,400, so it overstated taxable income at $86,130."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It subtracted the $1,400 of auto loan interest as a deduction, lowering taxable income to $83,630. That interest is not qualified passenger-vehicle loan interest under OBBBA because no qualifying new, U.S.-assembled vehicle loan is established, so taxable income is $85,030 and tax is $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It inflation-guessed a $15,450 standard deduction and brackets of $12,283/$49,929. The actual 2026 OBBBA values are a $16,100 deduction and $12,400/$50,400 thresholds, so its tax came out at $13,611.04."
-us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It used the correct 2026 brackets but a $15,800 standard deduction instead of $16,100. The extra $300 of taxable income at 22% adds $66, which turns $13,418.60 into $13,484.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated the $2,308 tax-exempt private pension as taxable and assumed net capital gains, when the $1,870 long-term capital loss actually reduces AGI to $101,130. Both errors inflated taxable income above the correct $85,030."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It claimed itemized medical effects, but $2,520 of medical costs falls far below the 7.5%-of-AGI floor of about $7,585. Total itemized deductions of about $1,871 are well under the $16,100 standard deduction. Its $9,587 understates the correct $13,418.60 on $85,030 of taxable income."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied 'scheduled' pre-OBBBA 2026 law with a personal exemption and a pre-TCJA standard deduction, leaving about $87,380 taxable under higher rates. OBBBA made the $16,100 standard deduction and 10/12/22% brackets permanent, giving $85,030 of taxable income and $13,418.60 of tax."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It subtracted a phased-down qualified auto-loan-interest deduction of $1,000, lowering taxable income to $84,030. The household's auto loan interest does not qualify for the OBBBA vehicle-loan interest deduction, so taxable income is $85,030 and tax is $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It deducted $1,000 of auto loan interest after phaseout on top of the $16,100 standard deduction. That interest is not qualified passenger-vehicle loan interest, so taxable income stays at $85,030 and tax is $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a $15,300 standard deduction instead of the 2026 OBBBA amount of $16,100, and 'projected' brackets instead of $12,400/$50,400. This inflated taxable income to $85,830 and tax to $13,774."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the 2026 TCJA sunset and applied an $8,400 standard deduction, a $5,300 exemption, and 10/15/25% brackets. OBBBA kept the TCJA rates and a $16,100 standard deduction, giving $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the 2025 pre-OBBBA $15,000 standard deduction and the 2025 brackets ($11,925/$48,475) instead of the 2026 values of $16,100 and $12,400/$50,400, so it taxed $86,130 instead of $85,030."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but used bracket thresholds of $12,200/$49,575 instead of the 2026 values of $12,400/$50,400. That moved too much income into the 22% bracket and produced $13,505.10."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied 'reverted TCJA' law with an $8,300 standard deduction, a $5,300 exemption, and 10/15/25% brackets. OBBBA made the $16,100 standard deduction and 10/12/22% brackets permanent for 2026, so the correct tax is $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but misapplied the bracket arithmetic and reported $13,426. The exact 2026 computation is $1,240 + $4,560 + 22% × $34,630 = $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value or explanation for this variable, so its response could not be scored."
-us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It took a $1,174 auto-loan-interest deduction (after phaseout), lowering taxable income to $83,856. The household's auto loan interest is not qualified vehicle-loan interest under OBBBA, so taxable income is $85,030 and tax is $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"Its own reasoning says the wages leave positive tax after the standard deduction, yet it reported $0, wrongly treating refundable-credit mechanisms as wiping out the liability. With no nonrefundable credits available, tax before refundable credits is $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used the 2025 OBBBA standard deduction of $15,750 instead of the 2026 amount of $16,100, giving taxable income of $85,380 instead of $85,030. Its tax of $13,560 also reflects inexact bracket thresholds."
-us,scenario_101,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but reported $13,487 instead of $1,240 + $4,560 + $7,618.60 = $13,418.60. That overstatement is consistent with bracket thresholds lower than the 2026 values of $12,400/$50,400."
-us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction and guessed brackets ($12,150/$49,000), computing $13,641. It then reported an unexplained $14,827 that matches none of its own figures."
-us,scenario_101,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It used a $13,600 pre-TCJA-style deduction and ignored the $1,870 capital loss deduction. It also taxed all income above $12,400 at 12%, skipping the 22% bracket, and then subtracted employee payroll tax as a fictitious nonrefundable credit."
+us,scenario_100,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Used the old 10% Montana EITC rate instead of the 20% rate for 2026. It briefly found the correct $2,056 net earned income, then switched to an unsupported $2,646. Its final $265 follows from neither figure; the correct result is 20% × $822.29 = $164.46."
+us,scenario_100,state_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Got the federal EITC right ($822.40 on 401(k)-reduced earned income) but applied the 2024–2025 Montana rate of 10%. The rate for 2026 is 20%, which gives about $164.46, not $82.24."
+us,scenario_100,state_refundable_credits,claude-fable-5.2,llm_error,state_local_rule,False,"Computed the federal EITC correctly at $822.40 but used the superseded 10% Montana match. The 2026 rate is 20%, so the credit is about $164.46, double its $82.24."
+us,scenario_100,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Said Montana has no refundable earned income credit. Montana's EITC is refundable and equals 20% of the federal EITC in 2026. This filer has two qualifying children and $2,055.72 of net earned income, giving a federal EITC of $822.29 and a Montana credit of $164.46."
+us,scenario_100,state_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Returned zero without applying Montana's refundable EITC. That credit equals 20% of the $822.29 federal EITC from 40% × $2,055.72 of 401(k)-reduced earned income, or $164.46."
+us,scenario_100,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Used the 10% Montana rate instead of 20%. It took 40% of gross wages ($5,915) without excluding the $3,859 traditional 401(k) deferral, which inflated the federal EITC to $2,366. Its own $237 result was then reported as $297."
+us,scenario_100,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Recognized that the federal EITC is positive and that Montana matches it, but still reported $0 as a 'conservative estimate'. It should have reported 20% × $822.29 = $164.46."
+us,scenario_100,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Treated the Montana EITC as nonrefundable and capped it at zero tax liability. Montana's EITC is refundable, so the full 20% × $822.29 = $164.46 is paid even with no state tax."
+us,scenario_100,state_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Computed the federal EITC correctly at $822.40 but used the old 10% Montana match. The 2026 rate is 20%, which yields about $164.46."
+us,scenario_100,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Applied the 10% Montana rate instead of 20% for 2026. It based the federal EITC on gross wages of $5,915 even though it noted the $3,859 traditional 401(k) deferral, which overstated the federal credit at $2,366 instead of $822.29."
+us,scenario_100,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Asserted that the Montana EITC is nonrefundable and reported zero. The credit is refundable and equals 20% of the $822.29 federal EITC, or $164.46."
+us,scenario_100,state_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Used the 10% Montana match instead of 20%. It applied it to a $2,366 federal EITC computed on gross wages, rather than $822.29 on earned income net of the $3,859 traditional 401(k) deferral."
+us,scenario_100,state_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Used a 12% Montana match, which is not a rate Montana has used; the 2026 rate is 20%. It also applied it to an unsupported $2,650 federal EITC. The correct federal EITC is 40% × $2,055.72 = $822.29."
+us,scenario_100,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Applied the 10% Montana rate instead of 20%. It also understated the federal EITC at $371, when 40% of $2,055.72 in 401(k)-reduced earned income is $822.29."
+us,scenario_100,state_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"Used the pre-2024 3% Montana EITC rate on $822.40, which gives $24.67, instead of the 20% rate for 2026. Its explanation also added a $1,200 young-child credit that the reference's Montana refundable total does not include, producing a stated $1,224.67 that does not match its submitted value."
+us,scenario_100,state_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Stated that no Montana refundable credits apply, missing the refundable Montana EITC. That credit equals 20% of the $822.29 federal EITC, or $164.46."
+us,scenario_100,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"Used the 10% Montana match instead of 20% for 2026. It applied it to a $2,366 federal EITC based on gross wages, ignoring the $3,859 traditional 401(k) exclusion that reduces earned income to $2,055.72."
+us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Said no state refundable credits apply, overlooking Montana's refundable EITC. That credit is 20% of the $822.29 federal EITC this filer earns with two qualifying children."
+us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Wrongly claimed that Montana has no refundable EITC and that the household has zero earned income. The head has $2,055.72 of earned income after 401(k) deferrals, which gives a federal EITC of $822.29 and a Montana credit of $164.46."
+us,scenario_100,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Concluded that the federal EITC is $0. A filer with two qualifying children and $2,055.72 of earned income is in the 40% phase-in range, so the federal EITC is $822.29. The Montana 20% match on that is $164.46."
+us,scenario_100,state_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Returned zero without applying Montana's refundable EITC. That credit equals 20% of the $822.29 federal EITC, or $164.46."
+us,scenario_100,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Used the correct 20% Montana rate but applied it to a $2,366 federal EITC based on gross wages. The $3,859 traditional 401(k) deferral is excluded from earned income, leaving $2,055.72 and a federal EITC of $822.29."
+us,scenario_100,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Applied the pre-2024 3% Montana EITC rate to a gross-wage federal EITC of $2,366, and the $70.98 in its explanation does not match its submitted $0. The correct result is 20% × $822.29 = $164.46."
+us,scenario_100,state_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Asserted that PolicyEngine does not model Montana refundable credits. The Montana EITC is computed at 20% of the $822.29 federal EITC, giving $164.46."
+us,scenario_100,state_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"Used the 10% Montana match instead of 20% for 2026. It applied it to a $2,366 federal EITC from gross wages instead of $822.29 from 401(k)-reduced earned income."
+us,scenario_100,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Incorrectly claimed Montana has no individual income tax. Montana taxes income and pays a refundable EITC of 20% of the federal EITC, which here is 20% × $822.29 = $164.46."
+us,scenario_100,state_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Applied the pre-2024 3% Montana EITC rate instead of the 20% rate for 2026. It used a $2,366 gross-wage federal EITC instead of $822.29 on earned income net of the $3,859 traditional 401(k) deferral."
+us,scenario_100,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Found no Montana refundable credits, missing the refundable Montana EITC. That credit equals 20% of the $822.29 federal EITC, or $164.46."
+us,scenario_100,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"Returned zero without applying Montana's refundable EITC. A low-income filer with two children and $2,055.72 of earned income receives 20% × $822.29 = $164.46."
+us,scenario_100,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Used the 10% Montana rate instead of 20%. It also overstated the federal EITC at $4,207, which is far above 40% of even gross wages. The correct federal EITC is 40% × $2,055.72 = $822.29."
+us,scenario_100,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Applied the superseded 10% Montana match to a $2,366 federal EITC based on gross wages. The correct inputs are the 20% rate for 2026 and $822.29 on 401(k)-reduced earned income."
+us,scenario_100,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Computed the Montana EITC as 10% of a $2,366 gross-wage federal EITC instead of 20% of $822.29. It also added a $1,200 young-child credit that is not part of the reference's Montana refundable credit total."
+us,scenario_100,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Used the 10% Montana EITC rate on a $2,366 federal EITC that ignores the $3,859 traditional 401(k) exclusion, instead of 20% of $822.29. It also added a $1,200 young-child credit that the reference total does not include."
+us,scenario_100,state_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Used the correct 20% EITC match but applied it to a $2,366 gross-wage federal EITC instead of $822.29 on 401(k)-reduced earned income. It also added $1,800 of child tax credit for both children, which is not in the reference's Montana refundable total."
+us,scenario_100,state_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Used the correct 20% Montana rate, but the $2,366 federal EITC came from 40% of gross wages. The $3,859 traditional 401(k) deferral is excluded from earned income, leaving $2,055.72 and a federal EITC of $822.29."
+us,scenario_100,state_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,"Applied the old 10% Montana match to a federal EITC of about $2,366 based on gross wages. The 2026 computation is 20% × $822.29 = $164.46."
+us,scenario_100,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Said no Montana refundable credits apply, overlooking the refundable Montana EITC. That credit is 20% of the $822.29 federal EITC."
+us,scenario_100,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"Wrongly asserted that Montana has no state EITC. Montana has a refundable EITC of 20% of the federal credit for 2026, which gives $164.46 here."
+us,scenario_100,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,Claimed Montana has no state EITC and looked only at elderly or disabled property-tax credits. The refundable Montana EITC of 20% × $822.29 = $164.46 applies.
+us,scenario_100,state_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Ruled out credits based on age and missing rent, and wrongly said Montana has no EITC here. The refundable Montana EITC equals 20% of the $822.29 federal EITC, or $164.46."
+us,scenario_100,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"Returned zero without applying Montana's refundable EITC. That credit is 20% of the $822.29 federal EITC, or $164.46."
+us,scenario_100,state_refundable_credits,inkling,llm_error,state_local_rule,False,"Used a 10% Montana EITC rate on a federal EITC of roughly $750, instead of 20% of $822.29. It also added a $1,200 child tax credit that is not part of the reference's Montana refundable credit total."
+us,scenario_100,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Asserted that Montana offers no state EITC. Montana's refundable EITC equals 20% of the federal EITC in 2026, which gives $164.46 on the $822.29 federal credit."
+us,scenario_100,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"Applied the outdated 10% Montana match to a $2,366 federal EITC based on gross wages. The correct calculation is 20% of $822.29 from earned income net of the $3,859 traditional 401(k) deferral."
+us,scenario_100,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Incorrectly stated that Montana has no state income tax. Montana's income tax includes a refundable EITC of 20% of the federal EITC, or $164.46 here."
+us,scenario_100,state_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"Used the 10% Montana match instead of 20%. It applied it to a $2,366 federal EITC computed on gross wages rather than $822.29 on $2,055.72 of 401(k)-reduced earned income."
+us,scenario_100,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,Treated Montana's earned income credit as nonrefundable. It is refundable and pays 20% × $822.29 = $164.46 regardless of tax liability.
+us,scenario_100,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Applied the 10% Montana rate instead of the 20% rate for 2026. It used a $2,366 federal EITC on gross wages without excluding the $3,859 traditional 401(k) deferral, which reduces the federal EITC to $822.29."
+us,scenario_100,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Wrongly asserted that Montana has no earned income tax credit. Montana's refundable EITC is 20% of the federal EITC in 2026, which gives $164.46 on the $822.29 federal credit."
+us,scenario_100,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Returned zero without applying Montana's refundable EITC. That credit equals 20% of the $822.29 federal EITC, or $164.46."
+us,scenario_100,tanf,claude-fable-5,llm_error,thresholds_rates,False,"Its own working found countable income (~$220/month) below the payment standard and a positive grant, yet it submitted $0 anyway. It also used the outdated $588 standard. The 2026 standard is about $725.10, which leaves $505.41/month."
+us,scenario_100,tanf,claude-fable-5.1,llm_error,categorical_eligibility,False,"It assumed Montana TANF is not modeled or paid. The engine does pay TANF to this 3-person unit: the ~$725.10 standard minus $219.69 in countable earnings gives $505.41/month, or $6,064.96 a year."
+us,scenario_100,tanf,claude-fable-5.2,llm_error,thresholds_rates,False,"Its method was right ($200 + 25% earned-income disregard, 3-person unit, resources under $3,000), but it rounded the payment standard to $725/month. The engine's 2026 standard is about $725.10/month, which gives $6,064.96 rather than $6,063.75."
+us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"It counted the household as 4 people when it is 3, and it used a $2,000 asset limit. Montana's TANF resource limit is $3,000, so $2,800 passes. It also never applied the earned-income disregard: $493/month in earnings leaves only $219.69 countable, well under the ~$725.10 standard."
+us,scenario_100,tanf,claude-haiku-5.5,llm_error,categorical_eligibility,False,"It returned $0 because no TANF receipt was listed, ignoring the instruction to assume take-up. The household meets the income and resource tests and gets about $505.41/month."
+us,scenario_100,tanf,claude-opus-4.7,llm_error,thresholds_rates,False,"It returned $0 'conservatively' without checking the benefit formula. Countable earnings after the $200 + 25% disregard are $219.69/month, well below the ~$725.10 three-person standard, so the grant is $505.41/month."
+us,scenario_100,tanf,claude-opus-4.8,llm_error,thresholds_rates,False,"It claimed a $595/month grant, which is above its own $588 maximum, so it never actually subtracted countable earnings. The correct figure is the ~$725.10 standard minus $219.69 in countable earnings, or $505.41/month."
+us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"It assumed the modeled rules pay nothing, but this unit qualifies. Countable earnings of $219.69/month are well below the ~$725.10 standard, which gives $6,064.96 a year."
+us,scenario_100,tanf,claude-opus-5.5,llm_error,thresholds_rates,False,"The disregard and resource steps were right, but it guessed a $730 payment standard. The engine's 2026 three-person standard is about $725.10, so the grant is $505.41/month, not $510."
+us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"It used a made-up ~$1,000 liquid-asset limit. Montana's TANF resource limit is $3,000, so the $2,800 bank balance passes and the household gets $505.41/month."
+us,scenario_100,tanf,claude-sonnet-5,llm_error,thresholds_rates,False,"It used a $327/month maximum and paid the full amount without subtracting countable earnings. The engine uses about $725.10 minus $219.69 in countable earnings (after the $200 + 25% disregard), or $505.41/month."
+us,scenario_100,tanf,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It assumed Montana TANF is not modeled. The engine computes a $6,064.96 annual grant for this 3-person unit."
+us,scenario_100,tanf,deepseek-v4-flash-0731,llm_error,asset_resource,False,"It added the $1,981 vehicle to bank assets. Montana excludes one vehicle, so countable resources are $2,800, under the $3,000 limit, and the household is eligible for $505.41/month."
+us,scenario_100,tanf,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $605 standard and a $200 + 50% disregard. Montana disregards $200 plus 25% of the rest, so countable earnings are $219.69/month (not $146.46), and the 2026 standard is about $725.10, giving $505.41/month."
+us,scenario_100,tanf,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It submitted $0 even though its own explanation worked out to $4,023.75. That figure also used a $555 standard instead of the ~$725.10 three-person standard, which with $219.69 in countable earnings gives $6,064.96."
+us,scenario_100,tanf,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a $500 maximum and a $200 + 50% disregard. The engine applies $200 + 25% (countable $219.69/month) against a ~$725.10 standard, giving $505.41/month."
+us,scenario_100,tanf,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The $200 + 25% disregard was right, but it used the outdated $588 three-person standard. The 2026 standard is about $725.10, so the grant is $505.41/month, not $368."
+us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"It said the household fails state requirements without naming any test. It is in fact eligible: resources are under $3,000 and countable earnings of $219.69/month are well below the ~$725.10 standard."
+us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"It assumed the head receives SSI, but no SSI is paid: $2,800 in bank assets exceeds the $2,000 SSI resource limit. The head therefore stays in a 3-person TANF unit with only earnings counted, for $505.41/month."
+us,scenario_100,tanf,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied a $200 + 50% disregard, and its $5,298 total matches the old $588 standard. The engine uses $200 + 25% (countable $219.69) against a ~$725.10 standard, giving $6,064.96."
+us,scenario_100,tanf,gemini-3.5-flash-lite,llm_error,categorical_eligibility,False,"It gave $0 with no reasoning. The correct result is the ~$725.10 standard minus $219.69 in countable earnings, or $505.41/month ($6,064.96 a year)."
+us,scenario_100,tanf,gemini-3.6-flash,llm_error,thresholds_rates,False,"Its $5,298.48 equals $588 minus ($292.92 × 50%), so it combined the outdated $588 standard with a 50% disregard. Montana's disregard is 25% and the 2026 standard is about $725.10."
+us,scenario_100,tanf,gemini-3.7-flash,llm_error,thresholds_rates,False,"It said the household qualifies and computed $4,824, then submitted $0. The $4,824 ($402/month) also understates the grant: the ~$725.10 standard minus $219.69 in countable earnings is $505.41/month."
+us,scenario_100,tanf,gemini-3.8-flash,llm_error,categorical_eligibility,False,"It said Montana TANF is not modeled, but the engine computes a $6,064.96 annual grant for this household."
+us,scenario_100,tanf,gemini-4-argon,llm_error,thresholds_rates,False,"Its $6,132 implies $511/month, which corresponds to a standard of about $731. The engine's 2026 standard is about $725.10, which minus $219.69 in countable earnings gives $505.41/month."
+us,scenario_100,tanf,glm-5.2,llm_error,other,False,"It added the $710 overtime premium on top of wages that already include it, and compared gross annual income to the payment standard without the $200 + 25% disregard. It also invented a work-requirement failure. Countable income is only $219.69/month, which leaves a $505.41 grant."
+us,scenario_100,tanf,glm-5.3,llm_error,categorical_eligibility,False,It assumed no positive benefit is modeled. The engine pays this 3-person unit about $725.10 − $219.69 = $505.41/month.
+us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It treated TANF as not payable because receipt wasn't listed, ignoring the take-up assumption. The household meets the income and resource tests and receives $6,064.96."
+us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It found no eligibility trigger even though a parent with two minor children, very low earnings and assets under $3,000 qualifies. The grant is $505.41/month."
+us,scenario_100,tanf,gpt-5.5,llm_error,thresholds_rates,False,"Its ~$150 countable income reflects a 50% disregard, and its $438 grant implies the old $588 standard. The engine applies $200 + 25% (countable $219.69) against ~$725.10."
+us,scenario_100,tanf,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $359.50/month grant implies a standard of about $579. The 2026 three-person standard is about $725.10, so the grant after $219.69 in countable earnings is $505.41/month."
+us,scenario_100,tanf,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $4,419.75 equals the outdated $588 standard minus $219.69 in countable earnings. The engine's 2026 standard is about $725.10, giving $6,064.96."
+us,scenario_100,tanf,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It counted SSI income that the head does not receive: $2,800 in assets exceeds the $2,000 SSI resource limit. With only earnings counted, the 3-person unit gets $505.41/month."
+us,scenario_100,tanf,gpt-6-astra,llm_error,thresholds_rates,False,"The disregard and resource logic were right, but it rounded the standard to $725/month. The engine's 2026 standard is about $725.10, which gives $6,064.96 instead of $6,063.75."
+us,scenario_100,tanf,gpt-6-luna,llm_error,thresholds_rates,False,"Its $5,298 matches the outdated $588 standard with a 50% disregard. Montana's disregard is $200 + 25%, and the 2026 standard is about $725.10, giving $6,064.96."
+us,scenario_100,tanf,gpt-6-sol,llm_error,thresholds_rates,False,"Its $368.25/month grant comes from the outdated $588 standard. The 2026 standard is about $725.10, which minus $219.69 gives $505.41/month."
+us,scenario_100,tanf,gpt-6.1-sol,llm_error,thresholds_rates,False,"It explicitly used a $588 three-person standard. The engine's 2026 standard is about $725.10/month, so the annual grant is $6,064.96, not $4,419.75."
+us,scenario_100,tanf,gpt-6.1-terra,llm_error,thresholds_rates,False,"Its $4,419.75 is the outdated $588 standard minus $219.69 in countable earnings. The 2026 standard is about $725.10, giving $6,064.96."
+us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,"It did no TANF calculation and defaulted to $0. The unit qualifies for the ~$725.10 standard minus $219.69 in countable earnings, or $505.41/month."
+us,scenario_100,tanf,grok-4.5,llm_error,thresholds_rates,False,"The disregard was right, but it used the outdated $588 standard. The 2026 three-person standard is about $725.10, so the grant is $505.41/month."
+us,scenario_100,tanf,grok-4.6,llm_error,thresholds_rates,False,"It used the outdated $588 payment standard. The engine applies about $725.10 for 2026, so the annual grant is $6,064.96."
+us,scenario_100,tanf,grok-4.7,llm_error,thresholds_rates,False,"The countable-income ($219.69) and resource logic were right, but it used the outdated $588 standard. The 2026 standard is about $725.10, giving $505.41/month."
+us,scenario_100,tanf,grok-build-0.1,llm_error,thresholds_rates,False,"It used a $675 maximum grant. The engine's 2026 three-person standard is about $725.10, which minus $219.69 gives $505.41/month."
+us,scenario_100,tanf,inkling,llm_error,thresholds_rates,False,"It used the outdated $588 maximum. The 2026 three-person standard is about $725.10, so the grant is $505.41/month, not $368."
+us,scenario_100,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no TANF value. The engine's figure is the ~$725.10 standard minus $219.69 in countable earnings, or $6,064.96 a year."
+us,scenario_100,tanf,kimi-k3,llm_error,thresholds_rates,False,"It handled the vehicle exclusion and the 25% disregard correctly but used the outdated $588 three-person grant. The 2026 standard is about $725.10, giving $6,064.96."
+us,scenario_100,tanf,minimax-m3,llm_error,thresholds_rates,False,"It assumed Montana benefits are only $50–$200/month and wiped them out with earnings. The standard is about $725.10, and countable earnings after the disregard are only $219.69, leaving $505.41/month."
+us,scenario_100,tanf,muse-spark-1.3,llm_error,thresholds_rates,False,"Its ~$434/month grant implies a standard of about $654. The 2026 three-person standard is about $725.10, which minus $219.69 gives $505.41/month."
+us,scenario_100,tanf,muse-wonder,llm_error,thresholds_rates,False,"Its $368/month implies the outdated $588 standard. The engine's 2026 standard is about $725.10, giving $505.41/month."
+us,scenario_100,tanf,ox-alpha,llm_error,categorical_eligibility,False,"It invented ~$780/month in SSI income. The head gets no SSI because $2,800 in assets exceeds the $2,000 SSI limit, so only $219.69 in countable earnings applies against the ~$725.10 standard."
+us,scenario_100,tanf,qwen-3.7-max,llm_error,other,False,"It counted the $710 overtime premium on top of wages that already include it, and compared gross annual income to an annual limit without the $200 + 25% disregard. Countable income is $219.69/month, well below the ~$725.10 standard."
+us,scenario_100,tanf,qwen3.8-max,llm_error,thresholds_rates,False,"It called $493/month in earnings too high. After the $200 + 25% disregard, countable income is $219.69/month, far below the ~$725.10 standard, so the grant is $505.41/month."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It took the full $1,400 auto loan interest as a deduction, which cut taxable income to $83,630 instead of $85,030. That deduction does not apply here. It then submitted $12,064, which matches none of its own bracket results ($13,111 or $13,058). The correct result is 22% on $34,630 above $50,400, plus $5,800, for $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It took a $1,000 qualified auto loan interest deduction ($1,400 less a $400 phaseout) on top of the $16,100 standard deduction. That gave taxable income of $84,030 instead of $85,030. The auto loan interest is not deductible, so its answer is exactly $220 (22% × $1,000) too low."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,thresholds_rates,False,"It subtracted a phased-down $1,000 auto loan interest deduction, giving taxable income of $84,030. Only the $16,100 standard deduction applies, so taxable income is $85,030. Its answer is low by 22% of $1,000, or $220."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It counted the $700 of veterans benefits in AGI, although they are excluded from gross income. It also used the 2024 standard deduction ($14,600) and 2024 brackets, and taxed nearly all income at 12% while skipping the 22% bracket. Its final $11,435 does not even match its own $10,637.50 figure, and it never applied $16,100 to AGI of $101,130."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,thresholds_rates,False,"It cut wages by pre-tax health premiums that are not excludable here and never did an exact bracket calculation, submitting a rounded guess of $13,900. The correct taxable income of $85,030 under the 2026 10/12/22% brackets ($12,400/$50,400) gives $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Its reasoning computed exactly $13,418.60 from $85,030 of taxable income and the correct 2026 brackets. It then submitted an unrelated $14,716, saying it was 'rounding to bracket estimates'. The error is a final-answer inconsistency, not a tax-rule error."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It correctly found $85,030 of taxable income and about $13,419 of tax using the $12,400/$50,400 thresholds. It then replaced that with an unsupported $13,876. The submitted value contradicts its own correct calculation."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It deducted the full $1,400 of auto loan interest as qualified passenger vehicle loan interest, cutting taxable income to $83,630. The facts never show a qualifying new, U.S.-assembled vehicle, so no deduction applies and taxable income is $85,030. Its answer is low by exactly 22% × $1,400, or $308."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used $15,750, the 2025 OBBBA standard deduction, instead of the 2026 figure of $16,100. It also guessed bracket thresholds of $12,300 and $49,950 instead of $12,400 and $50,400. That gave taxable income of $85,380 and tax of $13,543 instead of $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but used bracket thresholds of $12,150 and $49,400 instead of $12,400 and $50,400, which gave $13,523.60. It then submitted an unexplained $13,322 that matches none of its own figures."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It took an auto loan interest deduction of about $1,174, using a linear 20% phaseout, and reduced taxable income to $83,856. No auto loan interest deduction applies, so taxable income stays at $85,030 and tax is $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,600 standard deduction instead of the 2026 figure of $16,100. It also used bracket thresholds of $12,175 and $49,550 instead of $12,400 and $50,400, so it taxed $85,530 on the wrong schedule and got $13,618."
+us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It cut wages by $3,840 of 'pre-tax' health premiums, counting the single $1,920 premium twice and treating it as excludable when the stated wages are not reduced. It also used a $15,400 standard deduction and $12,200/$49,700 thresholds. The correct taxable income is $85,030."
+us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It assumed the TCJA expires in 2026, applying an $8,300 standard deduction, a $5,300 personal exemption and 10/15/25% brackets. OBBBA made the TCJA structure permanent: the 2026 single standard deduction is $16,100, there is no personal exemption, and the brackets are 10/12/22%."
+us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a pre-OBBBA projected standard deduction of $15,350 instead of $16,100. It also used $12,200/$49,575 thresholds instead of $12,400/$50,400, which gave taxable income of $85,780 and tax of $13,670.10."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It assumed the TCJA sunsets, applying an $8,300 standard deduction, a $5,300 personal exemption and 15%/25% brackets. It also subtracted $1,920 of ESI premiums from wages, though the stated wages are not reduced by them. Under OBBBA the 2026 calculation is $101,130 − $16,100 = $85,030, taxed at 10/12/22%."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It ignored the $1,870 capital loss that reduces AGI to $101,130, and used a $15,300 standard deduction instead of $16,100. That gave $87,700 of taxable income, and its $13,385 does not follow from correct 2026 brackets on any of its figures."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It found the right AGI of $101,130 but never applied the $16,100 standard deduction or the 2026 bracket schedule, submitting a rough $15,400. Exact computation on $85,030 of taxable income at 10% to $12,400, 12% to $50,400 and 22% above gives $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It applied a post-TCJA-sunset regime: an $8,250 standard deduction, a $5,300 personal exemption and 10/15/25% brackets. OBBBA kept the $16,100 standard deduction (2026, single), no personal exemption and 10/12/22% brackets."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It used a vague standard deduction of about $15,000 and did no explicit bracket calculation, submitting an unsupported $11,102. Correctly, $101,130 − $16,100 = $85,030, taxed at 10/12/22% with $12,400/$50,400 thresholds, gives $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It assumed the TCJA expires, using an $8,300 standard deduction, a $5,300 exemption and pre-TCJA 10/15/25% brackets. OBBBA made the $16,100 standard deduction and 10/12/22% brackets permanent for 2026."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It subtracted an $8,300 standard deduction and a $5,300 personal exemption under an assumed TCJA sunset, getting $87,530 of taxable income at higher pre-TCJA rates. OBBBA kept the 2026 single standard deduction at $16,100 with no exemption and 10/12/22% brackets."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used the pre-OBBBA 2025 standard deduction of $15,000 and the 2025 thresholds of $11,925 and $48,475. The 2026 values are $16,100, $12,400 and $50,400, so its taxable income was $86,130 instead of $85,030."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"It subtracted the full $1,400 of auto loan interest as a deduction, giving taxable income of $83,630. No auto loan interest deduction applies, so taxable income is $85,030. Its answer is low by exactly $308."
+us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It inflated older parameters to a $15,450 standard deduction and $12,283/$49,929 thresholds instead of the published 2026 values of $16,100, $12,400 and $50,400. That produced $85,680 of taxable income and $13,611.04 of tax."
+us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"Its brackets were correct ($12,400/$50,400), but it used a $15,800 standard deduction instead of the 2026 figure of $16,100. That overstated taxable income by $300 and tax by $66."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It counted the tax-exempt private pension as taxable income and treated the capital position as a small net gain, when it is a $1,870 loss that reduces AGI. Correct AGI is $101,130 and taxable income is $85,030, which gives $13,418.60, not $14,498."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"It cited unspecified 'itemized medical effects' and exclusions without any computation and submitted $9,587. Medical expenses of $2,520 are far below the 7.5%-of-AGI floor, so the $16,100 standard deduction applies to $101,130 of AGI and the tax is $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied 'scheduled 2026 current-law' TCJA-sunset parameters, with a standard deduction plus personal exemption that left $87,380 of taxable income at pre-TCJA rates. OBBBA made the $16,100 standard deduction and 10/12/22% brackets permanent, so taxable income is $85,030."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It took a phased-down qualified auto loan interest deduction of $1,000, reducing taxable income to $84,030. No auto loan interest deduction applies, so its result is low by exactly $220."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It subtracted a $1,000 auto loan interest deduction after the phaseout, on top of the $16,100 standard deduction. The household has no qualifying vehicle loan deduction, so taxable income is $85,030 and tax is $220 higher, at $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It used a projected $15,300 standard deduction instead of the OBBBA 2026 figure of $16,100, plus unspecified projected brackets. That gave $85,830 of taxable income and $13,774 instead of $85,030 and $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It assumed the TCJA sunsets, applying an $8,400 standard deduction, a $5,300 exemption and 10/15/25% brackets. OBBBA kept the 2026 single standard deduction at $16,100 with 10/12/22% brackets and no personal exemption."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used the 2025 pre-OBBBA $15,000 standard deduction and 2025 thresholds of $11,925 and $48,475. The 2026 values are $16,100, $12,400 and $50,400, so its taxable income came out $1,100 too high."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but used bracket thresholds of $12,200 and $49,575 instead of the 2026 values of $12,400 and $50,400. That moved too much income into the 22% bracket and gave $13,505.10."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It applied a 'reverted TCJA' regime: an $8,300 standard deduction, a $5,300 exemption and 15%/25% brackets. OBBBA made the $16,100 standard deduction and 10/12/22% brackets permanent for 2026."
+us,scenario_101,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It found the correct taxable income of $85,030 but did not compute the brackets exactly with the 2026 thresholds of $12,400 and $50,400, and submitted an approximate $13,426. The exact result is $1,240 + $4,560 + $7,618.60 = $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no answer and no reasoning for this variable, so nothing could be scored."
+us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It took an auto loan interest deduction of $1,174, reducing $1,400 by 20% of the $1,130 of AGI over $100,000, and cut taxable income to $83,856. No auto loan interest deduction applies, so taxable income is $85,030."
+us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"Its own reasoning says tax before credits is positive, yet it submitted 0, deferring to a nonexistent prior 'numeric answer'. With no nonrefundable credits, the tax on $85,030 of taxable income is $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used the 2025 OBBBA standard deduction of $15,750 instead of the 2026 figure of $16,100, giving $85,380 of taxable income. It also approximated the bracket tax at about $13,560 instead of computing it with the $12,400/$50,400 thresholds."
+us,scenario_101,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It reached the correct taxable income of $85,030 but never computed the brackets exactly, submitting an approximate $13,487. With the 2026 thresholds of $12,400 and $50,400 the tax is exactly $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction and $12,150/$49,000 thresholds instead of $16,100, $12,400 and $50,400, which gave $13,641. It then submitted an unrelated $14,827 that contradicts its own figure."
+us,scenario_101,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It used a $13,600 deduction instead of $16,100 and dropped the $1,870 capital loss by netting it against the tax-exempt pension. It also taxed everything above $12,400 at 12%, skipping the 22% bracket, and subtracted payroll tax as if it were a nonrefundable credit, which it is not."
us,scenario_101,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_101,payroll_tax,claude-haiku-4.5,llm_error,other,False,"It correctly computed $6,386 Social Security plus $1,493.50 Medicare = $7,879.50. It then submitted $8,149.50, citing a made-up 'adjustment for wage base cap and rounding.' The cap does not bind at $103,000 of wages, so no adjustment applies and the submitted value contradicts its own arithmetic."
us,scenario_101,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its components ($6,386 Social Security plus about $1,494 Medicare) add to $7,879.50, but it submitted $8,803. That figure does not follow from any of the components it listed, so this is an addition error. No extra payroll tax applies in Texas."
@@ -9221,57 +9320,58 @@ us,scenario_108,state_income_tax_before_refundable_credits,grok-build-0.1,llm_er
us,scenario_108,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"Gave no value and no explanation for state_income_tax_before_refundable_credits. The correct derivation counts only the $7,656 of wages as Wisconsin income, since the engine treats the $10,392 survivor benefits as nontaxable and Social Security is exempt, and the standard deduction takes taxable income to $0, so the answer is $0."
us,scenario_108,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"Added the $10,392 survivor benefits to wages as a 'taxable survivor pension', giving $18,048 of WI AGI. The engine's taxable base is $7,656 of wages, and the standard deduction reduces that to $0. Even on its own ~$2,900 of taxable income, the nonrefundable renter school property tax credit ($172.80) would have wiped out its $102."
us,scenario_108,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Taxed the survivor income on top of wages, leaving $2,680 taxable. The engine's Wisconsin base is only the $7,656 of wages, which the standard deduction fully covers. It also never applied the $172.80 nonrefundable school property tax credit for renters on $7,200 rent, which would have reduced its $94 to $0."
-us,scenario_108,state_refundable_credits,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits, reached $30,982, and said that was over the limit, then still reported a made-up partial credit of $636 that does not follow from any formula. PolicyEngine's steps are $20,590 of income (wages plus Social Security), $1,440 of rent-based property tax (20% of rent), and 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It added the $10,392 generic survivor benefits to homestead household income, got $30,982, and zeroed the credit for exceeding the $24,680 limit. PolicyEngine's wi_homestead_income counts only wages plus Social Security ($20,590), which gives a $271.39 homestead credit."
-us,scenario_108,state_refundable_credits,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It got PolicyEngine's $20,590 homestead income, then wrongly said the Wisconsin homestead credit is not a refundable income tax credit. The homestead credit is claimed on the Wisconsin return and is fully refundable, and at this income it equals 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"It reasoned that no Wisconsin tax liability means no refundable credits, which misunderstands refundability. The homestead credit is paid regardless of liability, and under PolicyEngine's reading it equals $271.39, based on $20,590 of income and $1,440 of rent-based property tax."
-us,scenario_108,state_refundable_credits,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It wrongly dropped the $12,934 of Social Security survivor benefits from homestead income (Wisconsin counts all Social Security) and added the generic survivor benefits instead, getting $18,048. It then guessed $300 instead of applying the formula. With $20,590 of income, 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to wages and Social Security, got $30,982 against the $24,680 homestead limit, and returned $0. PolicyEngine's homestead income is $20,590 (wages plus Social Security), which gives $271.39."
-us,scenario_108,state_refundable_credits,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It used about $18,000 of household income, which means it left out the $12,934 of Social Security that Wisconsin counts, and it gave $371 without showing a computation. PolicyEngine's homestead income is $20,590 (wages plus Social Security), and 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It treated homestead income as $30,982 by including the $10,392 generic survivor benefits, which is over the $24,680 limit. PolicyEngine counts only wages and Social Security ($20,590), so the homestead credit is $271.39."
-us,scenario_108,state_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It explicitly added the $10,392 other survivor benefits, got $30,982, and zeroed the homestead credit. PolicyEngine's wi_homestead_income is $20,590 (wages plus Social Security), which gives 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It said Wisconsin refundable credits require a qualifying child and never computed the homestead credit, which needs no child. An 85-year-old disabled renter with $20,590 of household income and $7,200 of rent gets a $271.39 homestead credit."
-us,scenario_108,state_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"It used PolicyEngine's $20,590 income but the wrong parameters: 25% of rent instead of 20% of rent ($1,440, below the $1,460 cap), plus an 8.65% phase-out rate and an $8,000 threshold instead of 8.785% and $8,060. PolicyEngine's result is 0.8 × (1,440 − 0.08785 × 12,530) = $271.39, and the model gave $297."
-us,scenario_108,state_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It concluded that household income exceeds the homestead limit, which only holds if the $10,392 generic survivor benefits are counted. PolicyEngine's homestead income is wages plus Social Security, $20,590, which gives a $271.39 credit."
-us,scenario_108,state_refundable_credits,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It used $30,982 of homestead income by including the $10,392 survivor benefits, so the credit came out to zero. PolicyEngine's wi_homestead_income is $20,590, which gives a $271.39 homestead credit."
-us,scenario_108,state_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It said household income exceeds the homestead limit, which depends on counting the $10,392 generic survivor benefits. PolicyEngine counts only wages and Social Security ($20,590), so the credit is 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It gave no analysis and never evaluated the Wisconsin homestead credit. Under PolicyEngine's reading the household qualifies for that refundable credit, which comes to $271.39 based on $20,590 of income and $1,440 of rent-based property tax."
-us,scenario_108,state_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It counted $30,982 of homestead income, which includes the $10,392 generic survivor benefits, against the $24,680 limit. PolicyEngine's homestead income is $20,590 (wages plus Social Security), which gives $271.39."
-us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It said the household is ineligible without computing anything. As an 85-year-old disabled renter with $20,590 of homestead income, the head qualifies for a $271.39 Wisconsin homestead credit."
-us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It got $30,982 of household income by including the $10,392 generic survivor benefits and zeroed the homestead credit. PolicyEngine counts $20,590 (wages plus Social Security), which gives a $271.39 credit."
-us,scenario_108,state_refundable_credits,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It applied the $24,680 limit to $30,982 of income that includes the $10,392 survivor benefits. PolicyEngine's wi_homestead_income is $20,590, which gives 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It said the head is ineligible for the homestead credit without giving any income or rent computation. With $20,590 of homestead income and $1,440 of rent-based property tax, the head gets a $271.39 refundable homestead credit."
-us,scenario_108,state_refundable_credits,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 generic survivor benefits to reach $30,982, which is over the $24,680 threshold. PolicyEngine's homestead income is wages plus Social Security ($20,590), which gives a $271.39 credit."
-us,scenario_108,state_refundable_credits,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It said household income exceeds the homestead limit, which depends on counting the $10,392 generic survivor benefits. PolicyEngine's $20,590 homestead income (wages plus Social Security) gives a $271.39 credit."
-us,scenario_108,state_refundable_credits,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It said homestead income exceeds the $24,680 limit, which only happens if the $10,392 survivor benefits are counted. PolicyEngine counts $20,590 of income, so the credit is 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"It used $30,982 of household income, including the $10,392 generic survivor benefits, against the $24,680 limit. PolicyEngine's wi_homestead_income is $20,590, which gives a $271.39 homestead credit."
-us,scenario_108,state_refundable_credits,glm-5.2,prompt_ambiguity,state_local_rule,False,"It included the $10,392 survivor benefits in homestead income ($30,982) and zeroed the credit. PolicyEngine counts only wages and Social Security ($20,590), so the credit is $271.39."
-us,scenario_108,state_refundable_credits,glm-5.3,prompt_ambiguity,state_local_rule,False,"It treated the $10,392 survivor benefits as countable pension income, reaching about $30,982 and going over the limit. PolicyEngine's homestead income is $20,590 (wages plus Social Security), which gives 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It said no Wisconsin refundable credit is triggered and never evaluated the homestead credit. Under PolicyEngine's reading the head qualifies as an 85-year-old disabled renter, and the credit is $271.39 based on $20,590 of income and $1,440 of rent-based property tax."
-us,scenario_108,state_refundable_credits,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It looked only for EITC-type refundable credits and missed the Wisconsin homestead credit. Based on $7,200 of rent (20% = $1,440) and $20,590 of household income, that credit is $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.5,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits in homestead income and concluded the household is over the limit. PolicyEngine counts only wages plus Social Security ($20,590), which gives a $271.39 credit."
-us,scenario_108,state_refundable_credits,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It assumed refundable Wisconsin credits require qualifying children and overlooked the homestead credit, which is available to elderly and disabled renters. At $20,590 of income and $1,440 of rent-based property tax, that credit is $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"It placed household income above the homestead limit, which depends on including the $10,392 generic survivor benefits. PolicyEngine's homestead income is $20,590, which gives $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It reported that no refundable credit applies and never evaluated the homestead credit. The head's $7,200 of rent and $20,590 of homestead income produce a $271.39 refundable homestead credit."
-us,scenario_108,state_refundable_credits,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"It counted $30,982 of income, including the $10,392 survivor benefits, against a $24,480 ceiling. PolicyEngine's homestead income is $20,590 (wages plus Social Security), which gives 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It said household income exceeds the homestead limit, which reflects including the $10,392 generic survivor benefits. PolicyEngine counts $20,590, which gives a $271.39 homestead credit."
-us,scenario_108,state_refundable_credits,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"It treated total household income, including the $10,392 survivor benefits, as over the homestead limit. PolicyEngine's wi_homestead_income is $20,590, which gives a $271.39 credit."
-us,scenario_108,state_refundable_credits,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"It put total household income over the homestead limit by counting the $10,392 generic survivor benefits. PolicyEngine's $20,590 homestead income (wages plus Social Security) gives $271.39."
-us,scenario_108,state_refundable_credits,gpt-6.1-terra,prompt_ambiguity,state_local_rule,False,"It said household income exceeds the homestead limit, which requires counting the $10,392 survivor benefits. PolicyEngine counts $20,590, so the credit is 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,grok-4.3,prompt_ambiguity,state_local_rule,False,"It said no state refundable credits qualify without evaluating the Wisconsin homestead credit. Under PolicyEngine's reading the household qualifies, and the credit is $271.39 based on $20,590 of income and $1,440 of rent-based property tax."
-us,scenario_108,state_refundable_credits,grok-4.5,prompt_ambiguity,state_local_rule,False,"It used about $30,982 of homestead income, which includes the $10,392 survivor benefits, against an approximately $25,000 limit. PolicyEngine's homestead income is $20,590, which gives a $271.39 credit."
-us,scenario_108,state_refundable_credits,grok-4.6,prompt_ambiguity,state_local_rule,False,"It included the $10,392 survivor benefits to reach $30,982 and zeroed the homestead credit. PolicyEngine counts only wages plus Social Security ($20,590), which gives $271.39."
-us,scenario_108,state_refundable_credits,grok-4.7,prompt_ambiguity,state_local_rule,False,"It explicitly added the $10,392 other survivor benefits to wages and Social Security and went over the homestead limit. PolicyEngine's wi_homestead_income is $20,590, which gives 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It called its $30,982 figure 'AGI + SS' but got it by putting the $10,392 survivor benefits into AGI, then compared it to a projected $26,500 limit. PolicyEngine's homestead income is $20,590, which gives a $271.39 credit."
-us,scenario_108,state_refundable_credits,inkling,prompt_ambiguity,state_local_rule,False,"It considered only the Wisconsin EITC and never evaluated the homestead credit. Under PolicyEngine's reading the 85-year-old disabled renter qualifies, and the credit is $271.39 based on $20,590 of income and $1,440 of rent-based property tax."
-us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_refundable_credits, so there was nothing to score against the $271.39 homestead credit."
-us,scenario_108,state_refundable_credits,kimi-k3,prompt_ambiguity,state_local_rule,False,"It used a broad household income of about $30,982, including the $10,392 survivor benefits, and zeroed the homestead credit. PolicyEngine counts $20,590 (wages plus Social Security), which gives $271.39."
-us,scenario_108,state_refundable_credits,minimax-m3,prompt_ambiguity,state_local_rule,False,"It reasoned that with no Wisconsin tax liability no refundable credits apply, which is backwards: refundable credits are paid even when there is no liability. PolicyEngine's homestead credit here is $271.39."
-us,scenario_108,state_refundable_credits,muse-spark-1.3,prompt_ambiguity,state_local_rule,False,"It said income was too high for the homestead credit, which follows from counting the $10,392 generic survivor benefits. PolicyEngine's homestead income is $20,590, which gives $271.39."
-us,scenario_108,state_refundable_credits,muse-wonder,prompt_ambiguity,state_local_rule,False,"It placed household income over the homestead limit by including the $10,392 survivor benefits. PolicyEngine counts $20,590, so the credit is 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,ox-alpha,prompt_ambiguity,state_local_rule,False,"It explicitly summed wages, Social Security survivor benefits, and the $10,392 other survivor benefits to get $30,982, which is over the limit. PolicyEngine's wi_homestead_income is $20,590, which gives a $271.39 credit."
-us,scenario_108,state_refundable_credits,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to get $30,982 against the $24,680 homestead limit. PolicyEngine counts only wages and Social Security ($20,590), which gives 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
-us,scenario_108,state_refundable_credits,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It tied Wisconsin refundable credits to having qualifying children and skipped the homestead credit. That credit is $271.39 for this elderly, disabled renter with $20,590 of homestead income."
+us,scenario_108,state_refundable_credits,claude-fable-5,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to homestead household income ($30,982) and concluded the household was over the limit. It then submitted an unsupported $636 that contradicts its own reasoning. With income of $20,590 (wages plus Social Security only) and rent at 20% ($1,440), the credit is 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
+us,scenario_108,state_refundable_credits,claude-fable-5.1,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits in homestead household income, reaching $30,982 and a $0 credit. PolicyEngine's homestead income is only wages plus Social Security ($20,590), which is under the $24,680 limit and gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,claude-fable-5.2,prompt_ambiguity,state_local_rule,False,"Its $20,590 income, 8.785% rate and $8,060 threshold match the reference. The error is the rent conversion: it used 25% of rent capped at $1,460 instead of 20% of rent ($1,440). That overstated the credit by 0.8 × $20, giving $287.39 instead of $271.39."
+us,scenario_108,state_refundable_credits,claude-haiku-4.5,prompt_ambiguity,state_local_rule,False,"It wrongly classified the Wisconsin homestead credit as non-refundable property tax relief and left it out. The homestead credit is a refundable credit claimed on the Wisconsin return. On the $20,590 income it had already computed, the credit is $271.39."
+us,scenario_108,state_refundable_credits,claude-haiku-5.5,prompt_ambiguity,state_local_rule,False,"It assumed refundable credits require Wisconsin tax liability to offset. The refundable homestead credit is paid regardless of liability: $271.39 based on $1,440 rent-equivalent property tax and $20,590 household income."
+us,scenario_108,state_refundable_credits,claude-opus-4.7,prompt_ambiguity,state_local_rule,False,"It reversed the income components, excluding Social Security and including the $10,392 survivor benefits ($18,048). It then guessed $300 instead of applying the formula. The correct homestead income is wages plus Social Security, $20,590, and 0.8 × (1,440 − 0.08785 × (20,590 − 8,060)) = $271.39."
+us,scenario_108,state_refundable_credits,claude-opus-4.8,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to wages and Social Security to get $30,982 and ruled the household over the $24,680 limit. Homestead income is only $20,590 (wages plus Social Security), which gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,claude-opus-5,prompt_ambiguity,state_local_rule,False,"It used household income of about $18,000, which drops the $12,934 of Social Security and keeps the survivor benefits, and reported an unsupported $371. The correct income is $20,590 (wages plus Social Security), and 0.8 × (1,440 − 8.785% × 12,530) = $271.39."
+us,scenario_108,state_refundable_credits,claude-opus-5.5,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits, putting homestead income at $30,982, over the limit. Homestead income is $20,590 (wages plus Social Security), below $24,680, so the credit is $271.39."
+us,scenario_108,state_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,state_local_rule,False,"It included the $10,392 survivor benefits in homestead household income ($30,982) and denied the credit. Only wages and Social Security ($20,590) count, which is under $24,680 and yields a $271.39 credit."
+us,scenario_108,state_refundable_credits,claude-sonnet-5,prompt_ambiguity,state_local_rule,False,"It dismissed the homestead credit without computing it. An 85-year-old disabled renter with $7,200 rent and $20,590 homestead income qualifies for a refundable credit of 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
+us,scenario_108,state_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,state_local_rule,False,"Its $20,590 income is correct, but it used 25% of rent capped at $1,460 instead of 20% of rent ($1,440). It also used a 8.65% rate over $8,000 instead of 8.785% over $8,060. The resulting $297 overshoots the correct $271.39."
+us,scenario_108,state_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,state_local_rule,False,"It ruled household income above the homestead limit, which means it counted the $10,392 survivor benefits. Homestead income is only wages plus Social Security ($20,590), which qualifies for a $271.39 credit."
+us,scenario_108,state_refundable_credits,deepseek-v4-pro,prompt_ambiguity,state_local_rule,False,"It used $30,982 of homestead income by adding the survivor benefits. The engine counts only $20,590 of wages and Social Security, which is under the limit and gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,state_local_rule,False,"It concluded income exceeded the homestead limit by counting the $10,392 survivor benefits. Homestead income is $20,590, and the credit is 0.8 × (1,440 − 8.785% × 12,530) = $271.39."
+us,scenario_108,state_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"It gave no reasoning and missed the Wisconsin homestead credit. With $7,200 rent (property tax equivalent $1,440) and $20,590 household income, the credit is $271.39."
+us,scenario_108,state_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,state_local_rule,False,"It computed homestead income as $30,982 by including the $10,392 survivor benefits. Only wages and Social Security count ($20,590), which is below $24,680 and yields $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,state_local_rule,False,"It declared the household ineligible for Wisconsin refundable credits without computing the homestead credit. On $20,590 of household income and $1,440 of rent-equivalent tax, that credit is $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"It used $30,982 of homestead income, which includes the $10,392 survivor benefits, and denied the credit. Homestead income is $20,590 (wages plus Social Security), and the credit is $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.5-flash,prompt_ambiguity,state_local_rule,False,"It added the survivor benefits to reach $30,982 and called the household ineligible. The engine's homestead income is $20,590, under the $24,680 limit, which produces a $271.39 credit."
+us,scenario_108,state_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,state_local_rule,False,"It asserted homestead ineligibility without computing it. The 85-year-old disabled renter has $20,590 of homestead income and $1,440 of rent-equivalent property tax, which gives a refundable credit of $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.6-flash,prompt_ambiguity,state_local_rule,False,"It counted the $10,392 survivor benefits, putting income at $30,982, over the $24,680 limit. Homestead income is only wages plus Social Security ($20,590), which yields $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.7-flash,prompt_ambiguity,state_local_rule,False,"It found income over the homestead limit by including the survivor benefits. The correct homestead income is $20,590, and 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.8-flash,prompt_ambiguity,state_local_rule,False,"It concluded income exceeded the $24,680 homestead limit by counting the $10,392 survivor benefits. Homestead income is $20,590, under the limit, so the credit is $271.39."
+us,scenario_108,state_refundable_credits,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"It used $30,982 of homestead income including the survivor benefits. The engine counts only wages and Social Security ($20,590), which gives a $271.39 homestead credit."
+us,scenario_108,state_refundable_credits,glm-5.2,prompt_ambiguity,state_local_rule,False,"It added the survivor benefits to reach $30,982 and denied the homestead credit. Homestead income is $20,590, below the limit, which produces a $271.39 credit."
+us,scenario_108,state_refundable_credits,glm-5.3,prompt_ambiguity,state_local_rule,False,"It treated the $10,392 survivor benefits as countable pension income, reaching $30,982, over the limit. Homestead income is only wages plus Social Security ($20,590), and the credit is $271.39."
+us,scenario_108,state_refundable_credits,gpt-5.4-mini,prompt_ambiguity,state_local_rule,False,"It missed the refundable Wisconsin homestead credit entirely. With $7,200 rent (20% equals $1,440) and $20,590 household income, the credit is $271.39."
+us,scenario_108,state_refundable_credits,gpt-5.4-nano,prompt_ambiguity,state_local_rule,False,"It looked only for EITC-like credits and missed the refundable homestead credit. That credit is 0.8 × (1,440 − 8.785% × (20,590 − 8,060)) = $271.39."
+us,scenario_108,state_refundable_credits,gpt-5.5,prompt_ambiguity,state_local_rule,False,"It included the listed survivor benefits in homestead household income and found the household over the limit. Homestead income is $20,590 (wages plus Social Security), which gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,gpt-5.6-luna,prompt_ambiguity,state_local_rule,False,"It assumed Wisconsin's refundable credits all depend on children and ignored the homestead credit for elderly and disabled renters. That credit is $271.39 on $20,590 of homestead income and $1,440 of rent-equivalent tax."
+us,scenario_108,state_refundable_credits,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"It ruled income above the homestead limit by counting the $10,392 survivor benefits. The engine's homestead income is $20,590, under $24,680, which yields $271.39."
+us,scenario_108,state_refundable_credits,gpt-5.6-terra,prompt_ambiguity,state_local_rule,False,"It found no refundable credit eligibility and overlooked the homestead credit. An 85-year-old disabled renter with $20,590 of homestead income receives a $271.39 credit."
+us,scenario_108,state_refundable_credits,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"It used $30,982 of income, including the survivor benefits, against a misstated $24,480 ceiling. Homestead income is $20,590 (wages plus Social Security), below the $24,680 limit, and the credit is $271.39."
+us,scenario_108,state_refundable_credits,gpt-6-luna,prompt_ambiguity,state_local_rule,False,"It concluded homestead income was over the limit by including the $10,392 survivor benefits. Only wages and Social Security count ($20,590), which yields a $271.39 credit."
+us,scenario_108,state_refundable_credits,gpt-6-sol,prompt_ambiguity,state_local_rule,False,"It counted the survivor benefits and found income over the homestead limit. Homestead income is $20,590, and 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
+us,scenario_108,state_refundable_credits,gpt-6.1-sol,prompt_ambiguity,state_local_rule,False,"It treated total income, including the $10,392 survivor benefits, as homestead income and denied the credit. Wages plus Social Security ($20,590) is under the limit and gives $271.39."
+us,scenario_108,state_refundable_credits,gpt-6.1-terra,prompt_ambiguity,state_local_rule,False,"It ruled income over the homestead limit by counting the survivor benefits. Homestead income is $20,590, which produces a $271.39 refundable credit."
+us,scenario_108,state_refundable_credits,grok-4.3,prompt_ambiguity,state_local_rule,False,"It found no qualifying credits and missed the Wisconsin homestead credit. On $1,440 of rent-equivalent tax and $20,590 of homestead income, that credit is $271.39."
+us,scenario_108,state_refundable_credits,grok-4.5,prompt_ambiguity,state_local_rule,False,"It used $30,982 of homestead income by adding the survivor benefits. Homestead income is $20,590, under the $24,680 limit, which gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,grok-4.6,prompt_ambiguity,state_local_rule,False,"It included the $10,392 survivor benefits, reaching $30,982, above the limit. Only wages and Social Security count ($20,590), and the credit is $271.39."
+us,scenario_108,state_refundable_credits,grok-4.7,prompt_ambiguity,state_local_rule,False,"It summed wages, Social Security and the other survivor benefits for homestead income. The engine excludes the $10,392 survivor benefits, leaving $20,590, which yields $271.39."
+us,scenario_108,state_refundable_credits,grok-build-0.1,prompt_ambiguity,state_local_rule,False,"It treated $30,982 (counting the $10,392 survivor benefits) as homestead income. The correct figure is $20,590 of wages plus Social Security, under the limit, which gives $271.39."
+us,scenario_108,state_refundable_credits,inkling,prompt_ambiguity,state_local_rule,False,"It considered only the Wisconsin EITC and never evaluated the refundable homestead credit. That credit is 0.8 × (1,440 − 8.785% × (20,590 − 8,060)) = $271.39."
+us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No value or explanation was returned for state_refundable_credits, so there is nothing to score. The correct output is the $271.39 Wisconsin homestead credit."
+us,scenario_108,state_refundable_credits,kimi-k3,prompt_ambiguity,state_local_rule,False,"It used broad household income of $30,982, including the $10,392 survivor benefits, and denied the homestead credit. Homestead income is $20,590, and the credit is $271.39."
+us,scenario_108,state_refundable_credits,minimax-m3,prompt_ambiguity,state_local_rule,False,"It assumed refundable credits require Wisconsin tax liability. The homestead credit is refundable and is paid without liability: $271.39 on $20,590 of homestead income."
+us,scenario_108,state_refundable_credits,muse-spark-1.3,prompt_ambiguity,state_local_rule,False,"It judged income too high for the homestead credit by counting the survivor benefits. Homestead income is $20,590 (wages plus Social Security), and the credit is $271.39."
+us,scenario_108,state_refundable_credits,muse-wonder,prompt_ambiguity,state_local_rule,False,"It found income over the homestead limit by including the $10,392 survivor benefits. The engine's $20,590 homestead income is below $24,680, which gives $271.39."
+us,scenario_108,state_refundable_credits,ox-alpha,prompt_ambiguity,state_local_rule,False,"It explicitly counted the $10,392 survivor benefits, reaching $30,982 of homestead income. Only wages and Social Security count ($20,590), which yields 0.8 × (1,440 − 0.08785 × 12,530) = $271.39."
+us,scenario_108,state_refundable_credits,qwen-3.7-max,prompt_ambiguity,state_local_rule,False,"It added the $10,392 survivor benefits to wages and Social Security ($30,982) and denied the homestead credit. Homestead income is $20,590, under $24,680, which gives a $271.39 credit."
+us,scenario_108,state_refundable_credits,qwen3.8-max,prompt_ambiguity,state_local_rule,False,"It tied Wisconsin refundable credits to qualifying children and missed the homestead credit for elderly and disabled renters. That credit is $271.39 on $1,440 of rent-equivalent tax and $20,590 of income."
us,scenario_109,child1_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model checked only CHIP's age and upper-income limits and treated satisfying them as sufficient. It omitted the prior Medicaid screen: the age-9 child qualifies for Florida Medicaid in the OLDER_CHILD category, which makes the child ineligible for CHIP."
us,scenario_109,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model calculated MAGI and compared it only with CHIP's upper-income limit. It failed to test the lower Medicaid eligibility pathway first; at this income, the age-9 child is Medicaid-eligible in Florida and therefore excluded from CHIP."
us,scenario_109,child1_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model recognized that CHIP applies between Medicaid and CHIP income limits but incorrectly placed this household in that interval. The age-9 child remains within Florida's Medicaid limit for the OLDER_CHILD category, so the child never enters the CHIP coverage band."
@@ -9368,51 +9468,52 @@ us,scenario_109,self_employment_tax,gpt-5.4-mini,llm_error,other,False,"The mode
us,scenario_109,self_employment_tax,gpt-5.4-nano,llm_error,other,False,"The model set up 30000 × 0.9235 × 0.153 correctly but evaluated it as 4234 when the product is 4238.87, understating the tax by $4.87 — its number implies a net-earnings base of $27,673 instead of $27,705. It then compounded the confusion with an irrelevant Additional Medicare Tax digression and a stray discarded figure of 2793 before committing to the miscomputed 4234."
us,scenario_109,self_employment_tax,gpt-5.6-luna,llm_error,other,False,"The model named the correct base (92.35% of $30,000) and the correct combined 15.3% Social Security and Medicare rate, but reported the product as $4,241 when $27,705 × 0.153 = $4,238.87, which rounds to $4,239. The $2.13 overstatement is a multiplication slip presented as a rounding, equivalent to using a net-earnings base of $27,719."
us,scenario_109,self_employment_tax,minimax-m3,llm_error,other,False,"The model computed every component correctly — $27,705 net earnings, 12.4% Social Security of $3,435.42, 2.9% Medicare of $803.45 — and stated the total as approximately $4,238, then submitted $4,233, contradicting its own arithmetic by $5. Its own component figures sum to $4,238.87, so the submitted value is a transcription error rather than a rule or base mistake."
-us,scenario_109,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used outdated parameters ($225 standard deduction, $1,158 max allotment) and got about $626/month. It then dropped that result and cut the benefit to $3,768 for unspecified 'payroll/SE adjustments', but SNAP counts gross self-employment income less business costs and gives no deduction for SE tax. The correct calculation is $1,183 − 30%×$1,739 (rounded up to $522) = $661/month."
-us,scenario_109,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"It used the right inputs ($261 standard deduction, $1,739 net income, $1,183 max) but kept the unrounded $661.30/month. SNAP rounds the 30% expected contribution up to the next whole dollar ($521.70 → $522), which makes the benefit $661/month and $7,932/year, not $7,936."
-us,scenario_109,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It took half of SE tax off gross income (SNAP allows no such deduction) and used a $1,316 max allotment instead of $1,183. Its final $244/month does not follow even from its own figures. Correctly, net income is $2,500 − $500 − $261 = $1,739, and the benefit is $1,183 − $522 = $661/month."
-us,scenario_109,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"It wrongly treated the household as ineligible. Gross income of $2,500/month is far below 130% of the five-person poverty line (about $4,079/month), net income of $1,739 is below 100% of FPL, and $900 in assets passes the asset test. The benefit is $661/month."
-us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It guessed the five-person max allotment at about $1,613/month instead of the FY2026 figure of $1,183, and used a $240 standard deduction instead of $261. That overstated the benefit at $1,085/month instead of $661."
-us,scenario_109,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"It cut SE income by 7.65% for SE tax, which SNAP does not do, and applied an excess shelter deduction even though $800 rent is less than half of the $1,739 adjusted income ($869.50). That pushed the benefit far above the correct $661/month ($1,183 − $522)."
-us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,"It claimed an excess shelter deduction applies from the $9,600 rent, but $800/month is below half of the $1,739 adjusted income, so the deduction is zero. Its $424/month does not follow from the correct net income: $1,183 − 30%×$1,739 (rounded to $522) = $661/month."
-us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It projected the five-person max allotment at $1,017 from a wrong FY2025 base, when the FY2026 figure is $1,183. It also used the $228 standard deduction instead of $261 for a five-person household, which gave $485/month instead of $661."
-us,scenario_109,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It counted six members when the household has five, and took SE tax off income. It then concluded net income exceeds the limit even though its own numbers pass both tests. Net income is $1,739, far below the five-person 100% FPL limit of $3,137.50, so the benefit is $661/month."
-us,scenario_109,snap,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It used the right $261 standard deduction, $1,739 net income and $1,183 max, but annualized the unrounded $661.30. SNAP rounds the 30% contribution up to $522, so the benefit is $661/month and $7,932/year."
-us,scenario_109,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $350 standard deduction instead of the FY2026 five-person figure of $261, and a $1,185 max instead of $1,183. That understated net income at $1,650 instead of $1,739 and overstated the benefit at $690/month instead of $661."
-us,scenario_109,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $232 standard deduction instead of $261, overstating net income at $1,768 instead of $1,739. It also used a $1,184 max instead of $1,183 and rounded the contribution down instead of up, which gave $654/month instead of $661."
-us,scenario_109,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used the FY2025 five-person max allotment of $1,158 instead of the FY2026 figure of $1,183, and a $244 standard deduction instead of $261. That gave $631/month instead of $661."
-us,scenario_109,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It had the right $1,183 max but used a $250 standard deduction instead of $261. That understated net income at $1,750 instead of $1,739, raised the contribution to $525 instead of $522, and gave $658/month instead of $661."
-us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used net income of $1,750 (understated standard deduction) and a $695/month benefit, which implies a max allotment of about $1,220 instead of $1,183. The correct benefit is $1,183 − 30%×$1,739 (rounded to $522) = $661."
-us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $765/month is about $104 too high, which matches removing about $347 more from net income, as an invented shelter/utility deduction would. Rent of $800 is below half of $1,739 and no utility costs are listed, so there is no excess shelter deduction and the benefit is $661/month."
-us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied an excess shelter deduction, which pushed the benefit to $786/month. Rent of $800 is less than half of the $1,739 adjusted income and no utility expenses are listed (so no Florida SUA), so the shelter deduction is $0 and the benefit is $661/month."
-us,scenario_109,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used a standard deduction of about $250 instead of $261 (net $1,750 instead of $1,739). Its $665/month implies a max allotment of about $1,190 instead of $1,183, so it overstated the correct $661/month."
-us,scenario_109,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It returned $0 without computing anything, treating an eligible household as ineligible. Gross income of $2,500/month passes the 130% FPL test, net income is $1,739, and the benefit is $1,183 − $522 = $661/month."
-us,scenario_109,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It treated the $800 rent as a deduction, but only shelter costs above half of adjusted income are deductible. Since $800 < $869.50, the excess shelter deduction is $0, net income stays at $1,739, and the benefit is $661/month, not about $752."
-us,scenario_109,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $732/month is $71 too high, which matches taking about $237 extra off net income through a shelter deduction that does not exist here. Rent of $800 is below half of $1,739, so net income is $1,739 and the benefit is $661/month."
-us,scenario_109,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"Its $675/month is $14 above the correct $661, which matches an overstated max allotment (about $1,197 instead of $1,183) or an overstated standard deduction. The correct calculation is $1,183 − 30%×($2,500 − $500 − $261), with the contribution rounded up to $522."
-us,scenario_109,snap,gemini-4-argon,llm_error,thresholds_rates,False,"It used $299, the FY2026 standard deduction for households of 6 or more, instead of $261 for five people. That understated net income at $1,701 instead of $1,739 and gave $672/month instead of $661."
-us,scenario_109,snap,glm-5.2,llm_error,thresholds_rates,False,"It used a $236 standard deduction instead of $261 and a $1,155 max allotment instead of $1,183. It also kept the unrounded contribution, which gave $625.80/month instead of $661."
-us,scenario_109,snap,glm-5.3,llm_error,thresholds_rates,False,"It approximated the standard deduction at $258 instead of $261 and the max allotment at $1,180 instead of $1,183. That gave $657/month instead of $661 ($1,183 − $522)."
-us,scenario_109,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"It described the household as having very low net income and high shelter costs, but net income is $1,739 and the $800 rent produces no excess shelter deduction. Its $484.67/month is well below the correct $1,183 − $522 = $661/month."
-us,scenario_109,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"It assumed no take-up and returned $0, ignoring the instruction to assume program take-up. The household passes every test and receives $661/month."
-us,scenario_109,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"It reached $659/month instead of $661, which matches a slightly understated max allotment or a standard deduction other than $261. The correct inputs are net income of $1,739, a contribution rounded up to $522, and a $1,183 max."
-us,scenario_109,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"It stated $659/month instead of $661, so it used slightly wrong FY2026 parameters. The correct figures are a $261 standard deduction, a $1,183 max allotment, and a 30% contribution of $522 (rounded up from $521.70)."
-us,scenario_109,snap,gpt-6-astra,llm_error,thresholds_rates,False,"It used the right parameters ($261 standard deduction, $1,739 net income, $1,183 max) but annualized $661.30/month. SNAP rounds the expected contribution up to the whole dollar ($522), so the benefit is $661/month and $7,932/year."
-us,scenario_109,snap,gpt-6-luna,llm_error,thresholds_rates,False,"Its $8,045 per year is about $670/month, roughly $9 too high, and not a whole-dollar monthly benefit. That matches an understated standard deduction or overstated max allotment, against the correct $1,183 − $522 = $661/month."
-us,scenario_109,snap,gpt-6.1-sol,llm_error,thresholds_rates,False,"It computed $661.30/month from the right inputs but did not round the 30% contribution ($521.70) up to $522. That left it at $7,935.60 instead of $661 × 12 = $7,932."
-us,scenario_109,snap,grok-4.3,llm_error,thresholds_rates,False,"It claimed income and assets exceed SNAP limits, but $2,500/month is well under 130% FPL for five (about $4,079), net income of $1,739 is under 100% FPL, and $900 in assets passes the asset test. The benefit is $661/month."
-us,scenario_109,snap,grok-4.5,llm_error,thresholds_rates,False,"It projected a $1,213 max allotment instead of the FY2026 five-person figure of $1,183, and used net income of $1,744 instead of $1,739. That gave $690/month instead of $661."
-us,scenario_109,snap,grok-4.6,llm_error,thresholds_rates,False,"It used a max allotment of about $1,190 instead of $1,183, with net income of about $1,740. That gave $668/month instead of $661 ($1,183 − $522)."
-us,scenario_109,snap,grok-4.7,llm_error,thresholds_rates,False,"It used FY2025 parameters, a $254 standard deduction and a $1,158 max allotment, instead of the FY2026 figures of $261 and $1,183. It also skipped monthly contribution rounding, which gave $7,610.40 instead of $7,932."
-us,scenario_109,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It used the FY2025 standard deduction of $254 instead of $261 and a $1,189 max instead of $1,183. That gave $667/month instead of $661."
-us,scenario_109,snap,inkling,llm_error,thresholds_rates,False,"It used a standard deduction of about $250 instead of $261 and a max allotment of about $1,190 instead of $1,183. That gave $665/month instead of $661."
-us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It used a $300/month standard deduction instead of $261 and projected a $1,220 max allotment instead of $1,183. That gave $710/month instead of $661."
-us,scenario_109,snap,minimax-m3,llm_error,thresholds_rates,False,"It claimed an excess shelter deduction drives net income to about $0, but $800 rent is below half of the $1,739 adjusted income, so there is no shelter deduction. Its $489/month and $1,151 max are both wrong; the correct benefit is $1,183 − $522 = $661/month."
-us,scenario_109,snap,muse-spark-1.3,llm_error,thresholds_rates,False,"It used the FY2025 max allotment of $1,158 instead of the FY2026 figure of $1,183, and a standard deduction giving $1,742 net instead of $1,739. It also skipped contribution rounding, which gave $635.40/month instead of $661."
-us,scenario_109,snap,muse-wonder,llm_error,thresholds_rates,False,"It used the FY2025 max allotment of $1,158 instead of $1,183 and an overstated standard deduction of about $296 (net $1,704 instead of $1,739). That gave $647/month instead of $661."
-us,scenario_109,snap,ox-alpha,llm_error,thresholds_rates,False,"Its explanation added a roughly $400 Florida utility allowance to rent and produced a $356 excess shelter deduction, but no utility expenses are listed, so no SUA applies and the $800 rent alone is below half of adjusted income. It also used a $306 standard deduction instead of $261. The explanation reaches $9,400, contradicting the submitted $7,920 ($660/month); the correct benefit is $661/month."
-us,scenario_109,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It gave the full maximum allotment as if net income were zero, and used $962 instead of the FY2026 five-person max of $1,183. Net income is $1,739, which reduces the benefit by $522 to $661/month."
-us,scenario_109,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It treated the unit as four people instead of five ($998 max instead of $1,183) and assumed net income was zero. After the earned income and standard deductions, net income is still $1,739, so the benefit is $1,183 − $522 = $661/month."
+us,scenario_109,snap,claude-fable-5,llm_error,thresholds_rates,False,"It used outdated parameters: a $225 standard deduction instead of $261 and a $1,158 maximum instead of $1,183. That gave about $626 a month. It then dropped its own result for an unsupported $3,768 by citing 'payroll/SE adjustments,' but SNAP does not deduct self-employment tax from self-employment income. The correct result is $1,183 − $522 = $661 a month."
+us,scenario_109,snap,claude-fable-5.1,llm_error,thresholds_rates,False,"Its parameters were right ($261 standard deduction, $1,183 maximum, $1,739 net income), but it did not round the 30% contribution of $521.70 up to $522. It kept $661.30 a month instead of $661 and reported $7,936 instead of 12 × $661 = $7,932."
+us,scenario_109,snap,claude-fable-5.2,llm_error,thresholds_rates,False,"It computed net income of $1,739 and the $1,183 maximum correctly but left the benefit at $661.30 a month. It should have rounded the 30% contribution up to $522, giving a whole-dollar benefit of $661 and $7,932 a year rather than $7,935.60."
+us,scenario_109,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"It cut gross income by half of the self-employment tax, which SNAP does not deduct, and used a $1,316 maximum instead of $1,183. Its $244 monthly benefit does not follow even from its own $1,830 net income (1,316 − 549 = 767). The correct result is $1,183 − 30% of $1,739, or $661 a month."
+us,scenario_109,snap,claude-haiku-5.5,llm_error,thresholds_rates,False,"It denied SNAP without doing the calculation. Net income of $1,739 is far below the $3,137.50 net income limit, and $900 in assets is under the limit. The household qualifies for $661 a month."
+us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"It used a FY2026 maximum of about $1,613 for five people, which is far above the actual $1,183, and a $240 standard deduction instead of $261. That inflated the monthly benefit to $1,085 instead of $661."
+us,scenario_109,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"It reduced self-employment income by self-employment tax, used a $217 standard deduction instead of $261, and assumed a shelter deduction that pushed the benefit to about $953 a month. Rent of $800 is below half of the $1,739 adjusted income and no utilities are listed, so there is no shelter deduction. The benefit is $661 a month."
+us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,"It claimed an excess shelter deduction, yet its $424 monthly benefit implies counted net income of about $2,530 against the $1,183 maximum, as if almost no deductions were applied. The correct steps are the $500 earned income deduction and the $261 standard deduction, giving $1,739 net income, with no shelter deduction. That yields $661 a month."
+us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It estimated the FY2026 maximum for five at $1,017 from a mistaken $967 FY2025 base; the actual FY2026 figure is $1,183. It also used a $228 standard deduction instead of $261. That gave $485 a month instead of $661."
+us,scenario_109,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"It counted six people when the household has five and cut income by self-employment tax. It then said net income was above the limit, even though its own figures put income well below both the gross and net limits. The five-person household qualifies for $661 a month."
+us,scenario_109,snap,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It used the correct $261 deduction, $1,739 net income and $1,183 maximum but annualized the unrounded $661.30 benefit. It should have rounded the 30% contribution up to $522, giving $661 a month and $7,932 a year rather than $7,936."
+us,scenario_109,snap,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $350 standard deduction instead of the FY2026 five-person figure of $261, and a $1,185 maximum instead of $1,183. That understated net income at $1,650 instead of $1,739 and overstated the benefit at $690 a month instead of $661."
+us,scenario_109,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"It used a $232 standard deduction instead of $261 and a $1,184 maximum instead of $1,183. That raised net income to $1,768 and gave $654 a month instead of $661."
+us,scenario_109,snap,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It used outdated figures: a $244 standard deduction and a $1,158 maximum. The FY2026 values for five people are $261 and $1,183. This produced $631 a month instead of $661."
+us,scenario_109,snap,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It had the $1,183 maximum right but used a $250 standard deduction instead of $261. Net income became $1,750 instead of $1,739 and the contribution $525 instead of $522, so the benefit was $658 a month instead of $661."
+us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It took net income as $1,750, which implies a $250 standard deduction instead of $261. Its $695 monthly benefit implies a maximum of about $1,220 instead of the FY2026 figure of $1,183. The correct benefit is $661 a month."
+us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no calculation. Its $765 monthly benefit is $104 above the correct $661, which implies net income of about $1,393 instead of $1,739, consistent with adding a shelter deduction that does not apply. Only the $500 earned income deduction and the $261 standard deduction apply."
+us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It applied an excess shelter deduction and reached $786 a month. Rent of $800 is less than half of the $1,739 adjusted income ($869.50) and no utility costs are listed, so the shelter deduction is zero and the benefit is $661 a month."
+us,scenario_109,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"It took net income as $1,750, which implies a $250 standard deduction instead of $261. Its $665 monthly benefit implies a $1,190 maximum instead of $1,183. The correct figures give $1,183 − $522 = $661 a month."
+us,scenario_109,snap,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave $0 with no calculation. The household passes the gross, net, asset and work tests, and $1,183 − 30% of $1,739 net income gives $661 a month."
+us,scenario_109,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"It counted the $800 rent as a deduction and reached $752.50 a month. SNAP only deducts shelter costs above half of adjusted income, and $800 is below $869.50, so there is no deduction. The benefit is $661 a month."
+us,scenario_109,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $732 monthly benefit implies net income of about $1,503, about $236 less than the correct $1,739. That means it subtracted more than the $500 earned income deduction and $261 standard deduction allow, such as a shelter deduction that is zero here. The benefit is $661 a month."
+us,scenario_109,snap,gemini-3.8-flash,llm_error,thresholds_rates,False,"It brought the $800 rent into the calculation and reached $675 a month. Rent is below half of the $1,739 adjusted income, so there is no shelter deduction, and $1,183 − $522 = $661 a month."
+us,scenario_109,snap,gemini-4-argon,llm_error,thresholds_rates,False,"It applied the $299 standard deduction for six or more people instead of the $261 figure for five. That understated net income at $1,701 and overstated the benefit at $672 a month instead of $661."
+us,scenario_109,snap,glm-5.2,llm_error,thresholds_rates,False,"It used FY2025 figures: a $236 standard deduction and a $1,155 maximum. The FY2026 values are $261 and $1,183. It also skipped whole-dollar rounding, giving $625.80 a month instead of $661."
+us,scenario_109,snap,glm-5.3,llm_error,thresholds_rates,False,"It used approximations: a $258 standard deduction instead of $261 and a $1,180 maximum instead of $1,183. That gave about $657 a month instead of $661."
+us,scenario_109,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"It assumed very low net income and high shelter costs, yet reported only about $485 a month, which contradicts that premise. Net income is $1,739, there is no shelter deduction, and $1,183 − $522 = $661 a month."
+us,scenario_109,snap,gpt-5.4-nano,llm_error,thresholds_rates,False,"It assumed no take-up and returned $0, but the prompt says to assume program take-up when required. The household is eligible for $661 a month."
+us,scenario_109,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $659 monthly benefit matches the FY2025 five-person standard deduction of $254 instead of the FY2026 figure of $261: $1,183 − 30% of $1,746, rounded to $524, is $659. With $261, net income is $1,739, the contribution is $522 and the benefit is $661."
+us,scenario_109,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"Its $659 monthly benefit results from the outdated $254 standard deduction instead of the FY2026 five-person figure of $261. The correct $1,739 net income gives a $522 contribution and $661 a month."
+us,scenario_109,snap,gpt-6-astra,llm_error,thresholds_rates,False,"It used the correct $261 deduction and $1,183 maximum but annualized $1,183 − 30% × $1,739 = $661.30 without rounding. The contribution must round up to $522, giving $661 a month and $7,932 a year rather than $7,935.60."
+us,scenario_109,snap,gpt-6-luna,llm_error,thresholds_rates,False,"$8,045 is not a multiple of 12. It works out to about $670.42 a month, which means it did not produce a whole-dollar monthly benefit and used deduction or maximum figures other than $261 and $1,183. The correct benefit is $661 a month."
+us,scenario_109,snap,gpt-6.1-sol,llm_error,thresholds_rates,False,"It reached $661.30 a month from the correct parameters but did not round the 30% contribution up to $522. The benefit is $661 a month and $7,932 a year, not $7,935.60."
+us,scenario_109,snap,grok-4.3,llm_error,thresholds_rates,False,"It said income and assets exceed SNAP limits. In fact $2,500 gross a month is under the gross limit, $1,739 net is under the $3,137.50 net limit, and $900 in assets is under the asset test, so the household gets $661 a month."
+us,scenario_109,snap,grok-4.5,llm_error,thresholds_rates,False,"It projected a $1,213 maximum instead of the published FY2026 figure of $1,183 and put net income at about $1,744 instead of $1,739. That gave $690 a month instead of $661."
+us,scenario_109,snap,grok-4.6,llm_error,thresholds_rates,False,"It used a $1,190 maximum instead of $1,183 and rounded net income to $1,740. That gave $668 a month instead of $1,183 − $522 = $661."
+us,scenario_109,snap,grok-4.7,llm_error,thresholds_rates,False,"It used outdated figures: a $254 standard deduction and a $1,158 maximum, instead of the FY2026 values of $261 and $1,183. It also worked on an annual basis without monthly whole-dollar rounding, giving $7,610.40 instead of 12 × $661."
+us,scenario_109,snap,grok-build-0.1,llm_error,thresholds_rates,False,"It used the old $254 standard deduction and a $1,189 maximum instead of $261 and $1,183, and also miscalculated adjusted income as $1,740. That gave $667 a month instead of $661."
+us,scenario_109,snap,inkling,llm_error,thresholds_rates,False,"It used about $250 for the standard deduction instead of $261 and about $1,190 for the maximum instead of $1,183. That gave $665 a month instead of $661."
+us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"It estimated the standard deduction at $300 a month instead of $261 and projected a $1,220 maximum instead of $1,183. That understated net income at $1,700 and inflated the benefit to $710 a month instead of $661."
+us,scenario_109,snap,minimax-m3,llm_error,thresholds_rates,False,"It said a capped shelter deduction brings net income to about $0, yet gave a benefit well below its own $1,151 maximum, so its reasoning contradicts its answer. In fact rent produces no shelter deduction, net income is $1,739, and with the correct $1,183 maximum the benefit is $661 a month."
+us,scenario_109,snap,muse-spark-1.3,llm_error,thresholds_rates,False,"It used a $1,158 maximum instead of the FY2026 figure of $1,183 and net income of $1,742 instead of $1,739. It also skipped whole-dollar rounding, giving $635.40 a month instead of $661."
+us,scenario_109,snap,muse-wonder,llm_error,thresholds_rates,False,"It used a $1,158 maximum instead of $1,183. Its net income of $1,704 implies a standard deduction of about $296 instead of $261. That gave $647 a month instead of $661."
+us,scenario_109,snap,ox-alpha,llm_error,thresholds_rates,False,"Its explanation added a Florida utility allowance of about $400 even though no utility costs are listed, which created a $356 shelter deduction and a $9,400 result. It also used a $306 standard deduction instead of $261. The $7,920 it submitted ($660 a month) matches neither its own explanation nor the correct $661 a month."
+us,scenario_109,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"It treated the 20% earned income deduction as part of the gross income test and then awarded a $962 maximum benefit. The household does not receive the maximum: the FY2026 maximum is $1,183, minus a $522 contribution on $1,739 net income, for $661 a month."
+us,scenario_109,snap,qwen3.8-max,llm_error,thresholds_rates,False,"It treated the household as four people and assumed zero net income, giving the full $998 maximum. The household has five people, net income after the $500 and $261 deductions is $1,739, and the benefit is $1,183 − $522 = $661 a month."
us,scenario_109,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a federal minimum Medicaid pathway for any household member with no individual income and few assets. Florida requires this nondisabled adult to satisfy a covered category, and Medicaid MAGI eligibility uses household income rather than treating the spouse’s unlisted personal income as an isolated $0 eligibility base."
us,scenario_109,spouse_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly calculated that household income exceeds Florida’s parent/caretaker limit and explicitly concluded that the spouse is not eligible, but then submitted value 1. No alternative Medicaid category applies to reverse its own eligibility analysis."
us,scenario_109,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model improperly converted the presence of an infant into pregnancy or postpartum eligibility. An age-zero child does not establish that the spouse is pregnant, postpartum, or the child’s mother, and caretaker status alone fails because household income exceeds Florida’s parent/caretaker threshold."
@@ -9426,55 +9527,56 @@ us,scenario_109,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligib
us,scenario_109,spouse_wic_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model assumed the spouse was the postpartum mother of the infant. Because postpartum status is not listed, it is false under the prompt, leaving the spouse outside every adult WIC category."
us,scenario_109,spouse_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred postpartum or breastfeeding status from the spouse's age and the infant's presence. Neither fact establishes the spouse's own qualifying status, and the prompt explicitly makes those unlisted statuses false."
us,scenario_109,spouse_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model inferred recent postpartum status or current breastfeeding from the existence of a zero-year-old child. Those personal statuses were unlisted and therefore false, so the spouse fails categorical eligibility before the income threshold affects the result."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"Its own work reached $25,841 using the $16,100 standard deduction and 2026 brackets. That figure leaves out the 2026 non-itemizer cash charitable deduction of $387, which lowers the result to $25,748. It then submitted $21,120, which matches none of its own arithmetic."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used the old $15,000 standard deduction instead of the 2026 figure of $16,100. It allowed a $198 traditional IRA deduction that the active-participant phase-out removes, and it skipped the $387 non-itemizer charitable deduction. It then made up $1,300 of nonrefundable credits to reach $25,837, while the correct ordinary tax on $112,572 is $19,615, not its $21,009."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It said the federal tax was substantial but submitted 0 as a placeholder and never computed it. Done correctly, the result is $19,615 of ordinary tax plus $6,133 of preferential tax, or $25,748."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used guessed bracket thresholds ($12,150, $49,400, $105,450) instead of the 2026 OBBBA thresholds ($12,400, $50,400, $105,700). It also left out the $387 non-itemizer cash charitable deduction, so taxable income was $153,843 instead of $153,455 and the total came out at $25,951."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It allowed the $198 traditional IRA deduction, which phases out completely for an active participant at about $170k MAGI. It left out the $387 non-itemizer charitable deduction and put the top of the 22% bracket at $107,500 instead of $105,700. These errors partly cancel and land on $25,758."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It counted the preferential-rate tax twice: about $26,150 that already included tax on all income, plus another $6,133. It submitted $30,150, which does not follow from those figures. It also allowed the phased-out $198 IRA deduction and left out the $387 non-itemizer charitable deduction."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It took a $10,400 qualified overtime deduction by inferring overtime from the 50-hour week. No overtime income is listed, so it counts as $0 and there is no overtime deduction. It also allowed the phased-out $198 IRA deduction and left out the $387 non-itemizer charitable deduction."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a $15,750 standard deduction (the 2025 figure) instead of 2026's $16,100 and made up its own 2026 brackets instead of $12,400/$50,400/$105,700. It also left out the $387 non-itemizer cash charitable deduction, so ordinary tax was overstated."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It itemized using $5,331 of unreimbursed employee expenses above the 2% floor. Miscellaneous itemized deductions are permanently repealed for 2026, so the standard deduction plus the $387 charitable deduction applies. It also used 2025 brackets, and after computing $26,045 it switched to an unsupported $20,200."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It got AGI, the standard deduction and the brackets right but left out the 2026 non-itemizer cash charitable deduction ($387). Taxable income should be $153,455, not $153,843, which overstates tax by about $93."
-us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction instead of 2026's $16,100 and allowed the $198 IRA deduction, which is phased out. It left out the $387 non-itemizer charitable deduction and overstated ordinary tax at $20,174."
-us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It assumed TCJA had expired, using a $4,850 personal exemption, a $6,800 standard deduction and 2%-floor miscellaneous deductions. It also subtracted health premiums from wages, and its ordinary tax of $32,490 on about $101k is far above any 2026 bracket result. The 2026 law gives $16,100 standard plus $387 charitable, and $19,615 of ordinary tax."
-us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It applied pre-TCJA rules: an $8,650 standard deduction, a $5,450 personal exemption and miscellaneous itemized deductions. It also subtracted the $6,889 ESI premiums from wages and took the phased-out $198 IRA deduction. That cut taxable income to $136,818 instead of $153,455."
-us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It left out the 2026 non-itemizer cash charitable deduction ($387), so taxable income was $153,843 instead of $153,455. Everything else was right, giving $25,840.76 instead of $25,747.77."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed TCJA had expired. It took a $5,050 personal exemption and itemized $5,331 of 2%-floor employee expenses, both repealed for 2026, and then applied pre-TCJA brackets. The correct path is the $16,100 standard deduction plus the $387 non-itemizer charitable deduction under the 2026 brackets."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"Its $19,688 is roughly the ordinary-rate tax on about $112.6k alone. It left out the 15% tax on $40,884 of qualified dividends and long-term gains ($6,133), which must be added to reach $25,748."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It assumed TCJA had expired and used personal exemptions and itemized deductions. Personal exemptions are suspended in 2026, and this household uses the standard deduction plus the non-itemizer charitable deduction. It also subtracted pre-tax health premiums from wages, but wages after pre-tax contributions are $95,755 (only the 401(k) is excluded)."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It assumed TCJA had expired, with a $5,050 personal exemption and 2%-floor unreimbursed employee expenses inside $20,096 of itemized deductions. Both are repealed for 2026, where the $16,100 standard deduction plus the $387 charitable deduction applies. It then taxed the remainder under pre-TCJA brackets."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,other,False,"Its $19,679 is roughly the ordinary-rate tax alone ($19,615 on $112,572). It left out the 15% tax on $40,884 of preferential income ($6,133)."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,889 ESI premiums from wages and got AGI of $163,054. PolicyEngine reduces wages only by the $4,245 traditional 401(k), so AGI is $169,943. Its lower $22,415 follows from that understated base."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It used AGI of $163,054, having subtracted the $6,889 ESI premiums from wages. Correct AGI is $169,943, with only the $4,245 traditional 401(k) excluded, and the understated base drove the result down to $22,765."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used a $15,350 standard deduction instead of 2026's $16,100 and left out the $387 non-itemizer charitable deduction. That inflated ordinary taxable income to $113,709 (should be $112,572) and ordinary tax to $19,937."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It left out the 2026 non-itemizer cash charitable deduction ($387). Taxable income should be $153,455, not $153,843, and the result is $25,840.76 instead of $25,747.77."
-us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a guessed $15,400 standard deduction and 2025 bracket thresholds ($11,925/$48,475/$103,350) instead of 2026's $16,100 and $12,400/$50,400/$105,700. It also left out the $387 non-itemizer charitable deduction, which overstated ordinary tax at $20,125."
-us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It said the $40,884 of qualified dividends and long-term gains fell in the 0% bracket. Ordinary income of about $112.6k sits on top of the $49,450 0% threshold, so all of it is taxed at 15% ($6,133). It also left out the $387 non-itemizer charitable deduction."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,missing_output,False,"Its explanation says federal tax stays positive, but it submitted 0 without computing anything. The correct liability is $19,615 of ordinary tax plus $6,133 of preferential tax."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It itemized mortgage interest, real estate tax, medical and charity, which together fall below the $16,100 standard deduction. It gave no bracket work, and its $16,811 is about $8,900 too low. The correct figure is $19,615 ordinary tax on $112,572 plus 15% on $40,884."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It left out the 2026 non-itemizer cash charitable deduction of $387 and kept taxable income at $153,843 instead of $153,455. That produced $25,841 instead of $25,748."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It correctly applied the $387 non-itemizer cash charitable deduction but undercounted the bracket tax by about $137. Ordinary tax on $112,572 under 2026 brackets is $19,615 and preferential tax is $6,133, which totals $25,748, not $25,611."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It took only the $16,100 standard deduction and left out the 2026 non-itemizer cash charitable deduction ($387). That overstated taxable income by $387 and tax by about $93."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It left out the $387 non-itemizer cash charitable deduction that applies alongside the 2026 standard deduction. That yields $25,840.76 instead of $25,747.77."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It subtracted the $6,889 ESI premiums from wages (AGI $163,054 instead of $169,943). It also made up a $9,000 qualified overtime deduction even though no overtime income is listed. It correctly applied the traditional 401(k) exclusion, the standard deduction and the $387 charitable deduction."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It stopped at taxable income of $153,843 and left out the 2026 non-itemizer cash charitable deduction ($387). The correct base is $153,455, which gives $25,748."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It took an $8,400 phased-down overtime deduction even though overtime income is unlisted and therefore $0. It also left out the $387 non-itemizer charitable deduction. Together, these errors produced taxable income of $145,443; the correct base is $153,455."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It treated the $6,889 employer-sponsored insurance premiums as a pre-tax wage reduction, giving AGI of $163,054. Only the $4,245 traditional 401(k) reduces wages, so AGI is $169,943 and ordinary taxable income is $112,572, not $105,683."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It used only the standard deduction and left out the $387 non-itemizer charitable deduction. Its implied ordinary tax of about $20,726 is roughly $1,111 above the correct $19,615 on $112,572 under the 2026 brackets."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It guessed $24,500 based on itemized deductions and ""standard nonrefundable credits"" that do not exist for this filer. Itemizing loses to the $16,100 standard deduction, the $387 charitable deduction applies on top of it, and there are no credits, so the result is $25,748."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It assumed TCJA had expired, with a personal exemption of about $5,300, 2%-floor employee expenses and pre-TCJA brackets. For 2026, exemptions and miscellaneous itemized deductions are permanently repealed, and the $16,100 standard deduction plus the $387 charitable deduction applies."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules: a $5,300 personal exemption, 2%-floor employee expenses and 10/15/25% brackets. The 2026 law keeps the $16,100 standard deduction, adds the $387 non-itemizer charitable deduction and uses the 10/12/22/24% brackets."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It assumed TCJA had expired, taking a $5,450 personal exemption and $5,331 of miscellaneous employee expenses and using 10/15/25% brackets. Those rules do not apply in 2026. The correct base is $169,943 − $16,100 − $387."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It used pre-TCJA rules: a $5,387 personal exemption, miscellaneous itemized deductions and 10/15/25% brackets. It also deducted the phased-out $198 IRA contribution. Under 2026 law, the standard deduction plus the $387 charitable deduction applies, and AGI stays at $169,943."
-us,scenario_110,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized $5,331 of 2%-floor unreimbursed employee expenses, which are permanently repealed for 2026. Without them, itemized deductions fall below the $16,100 standard deduction plus $387 charitable. It also understated ordinary tax at $18,495."
-us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value and no explanation for federal_income_tax_before_refundable_credits, so there was nothing to score."
-us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It took an $8,425.50 qualified overtime deduction based on an assumed $10,400 premium, but no overtime income is listed, so it is $0. It also deducted the $198 traditional IRA contribution, which is phased out for an active participant at this income."
-us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,missing_output,False,"It falsely claimed that PolicyEngine returned 0 and did no calculation. The household has $153,455 of taxable income that produces $25,748 of tax."
-us,scenario_110,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used the standard deduction without the 2026 $387 non-itemizer charitable deduction. Its $26,360 implies about $20,227 of ordinary tax, which points to outdated standard-deduction or bracket figures instead of the 2026 $16,100 and $12,400/$50,400/$105,700 thresholds."
-us,scenario_110,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It reduced wages by the $6,889 employer health premiums, giving AGI of $163,054. Only the $4,245 traditional 401(k) is excluded, so AGI is $169,943, and the understated base produced $24,095."
-us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its own reasoning reached $26,973 but it submitted $43,879.20, which nothing in its work supports. It also failed to exclude the $4,245 traditional 401(k) from wages and left out the $387 non-itemizer charitable deduction."
-us,scenario_110,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It stated AGI as $113,617, which drops about $56k of interest, dividend, gain and estate income from the actual $169,943. It also took a QBI deduction with no qualified business income and made up $194.77 of nonrefundable credits."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own reasoning reached $25,840.76 but it submitted $21,120, which contradicts its own computation. The computation also left out the $387 non-itemizer cash charitable deduction, which lowers taxable income to $153,455 and tax to $25,748."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It made up a $7,116 qualified overtime deduction from the hours worked, but no overtime compensation input exists, so the deduction is zero. It also deducted the $198 traditional IRA contribution, which is fully phased out for an active 401(k) participant with MAGI near $170k. Both errors cut taxable income below the correct $153,455."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"It used a $15,000 standard deduction instead of the 2026 figure of $16,100, took the phased-out $198 IRA deduction, and left out the $387 non-itemizer charitable deduction. That gave $27,137, which it then cut by an invented $1,300 of nonrefundable credits even though no credit applies."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It never computed the tax and submitted $0 as a placeholder, even though it acknowledged a substantial liability. The correct calculation is $153,455 of taxable income, giving $19,615 of ordinary tax plus $6,133 at 15% on qualified dividends and long-term gains."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It guessed 2026 bracket breakpoints ($12,150/$49,400/$105,450) instead of the OBBBA-indexed $12,400/$50,400/$105,700, which overstated ordinary tax by about $110. It also left out the $387 non-itemizer cash charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It deducted the $198 traditional IRA contribution even though the deduction is phased out for an active 401(k) participant at this MAGI. It also left out the $387 non-itemizer charitable deduction and used a wrong 22% bracket top of about $107,500 instead of $105,700. The errors offset to $25,758, but the path is wrong."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It garbled the final sum by adding the $6,133 preferential tax on top of a figure that already included it, arriving at $30,150. It also took the phased-out $198 IRA deduction and left out the $387 non-itemizer charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It claimed a $10,400 overtime-premium deduction worked out from 50 weekly hours, but no qualified overtime compensation input exists, so the deduction is zero. It also took the phased-out $198 IRA deduction and left out the $387 non-itemizer cash charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used a guessed $15,750 standard deduction instead of the 2026 figure of $16,100 and inflated 2025 brackets by hand instead of using the 2026 breakpoints ($12,400/$50,400/$105,700). It also left out the $387 non-itemizer charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It deducted unreimbursed employee expenses as a 2%-floor miscellaneous itemized deduction, but OBBBA eliminated that deduction permanently. It also used 2025 brackets, then arbitrarily cut its own $26,045 result to $20,200 with no rule behind the cut."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It took the $16,100 standard deduction but left out the 2026 non-itemizer cash charitable deduction ($387 of cash gifts, within the $1,000 single cap). That kept taxable income at $153,843 instead of $153,455 and overstated tax by about $93."
+us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used the old $15,000 standard deduction instead of $16,100, deducted the phased-out $198 IRA contribution, and left out the $387 non-itemizer charitable deduction. Its ordinary tax of $20,174 on $113,861 also reflects bracket thresholds below the 2026 values."
+us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It excluded health premiums from wages and applied TCJA-sunset rules: a $6,800 standard deduction, a personal exemption, and a 2%-floor deduction for employee expenses. None of these apply under OBBBA's permanent rules. Its ordinary tax of $32,490 on about $101k is also arithmetically far too high."
+us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It excluded the $6,889 ESI premiums from wages and deducted the phased-out $198 IRA contribution. It also assumed the TCJA sunset ($8,650 standard deduction, $5,450 personal exemption, itemized $20,588), when the correct calculation uses AGI of $169,943, the $16,100 standard deduction plus $387 cash charity, and no exemption."
+us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It left out the $387 non-itemizer cash charitable deduction available to standard-deduction filers starting in 2026. Taxable income stayed at $153,843 instead of $153,455, so tax came out $93 too high."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA sunset: a $5,050 personal exemption, itemizing with $5,331 of 2%-floor employee business expenses, and pre-TCJA brackets. OBBBA made the TCJA rules permanent, so the correct deductions are the $16,100 standard deduction plus the $387 charity deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"$19,688 matches roughly the ordinary-rate tax alone (about $19,615–$19,708). That means it dropped the 15% tax on $40,884 of qualified dividends and long-term gains, which adds about $6,133."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It excluded pre-tax health premiums from wages and applied TCJA-sunset rules with personal exemptions and itemized deductions. Under OBBBA's permanent rules, AGI is $169,943, and the deductions are the $16,100 standard deduction plus the $387 non-itemizer charity deduction, with no exemption."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA sunset: a $5,050 personal exemption, 2%-floor employee business expenses, and pre-TCJA brackets. Under OBBBA's permanent rules, those deductions are gone, and taxable income is $153,455 after the $16,100 standard deduction and $387 cash charity."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"$19,679 roughly equals the ordinary-rate tax alone. That means it left out the 15% tax of about $6,133 on $40,884 of qualified dividends and long-term capital gains."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It built AGI of $163,054 by excluding the $6,889 employer-sponsored insurance premiums from wages, but the correct AGI is $169,943 with only the 401(k) deferral excluded. Its $22,415 also implies further unspecified deductions beyond the $16,100 standard deduction and $387 charity deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It used AGI of about $163,054 by excluding the $6,889 ESI premiums from wages, but the correct AGI is $169,943. Its $22,765 understates the correct $25,748, which comes from $16,100 standard plus $387 charity deductions."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It used an approximate $15,350 standard deduction instead of the 2026 figure of $16,100 and left out the $387 non-itemizer cash charitable deduction. Its ordinary tax also overshoots the 2026 brackets."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,It applied the standard deduction but left out the $387 non-itemizer cash charitable deduction that starts in 2026. That left taxable income $387 too high and tax $93 above the correct figure.
+us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used an estimated $15,400 standard deduction and 2025 bracket breakpoints instead of the 2026 values ($16,100 standard deduction; $12,400/$50,400/$105,700 brackets). It also left out the $387 non-itemizer charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It put the $40,884 of qualified dividends and long-term gains in the 0% bracket. But $112,959 of ordinary income sits beneath them, well above the roughly $49,450 0% threshold, so all of it is taxed at 15%, which adds about $6,133. It also left out the $387 charity deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"Its own reasoning said the tax stays positive, yet it submitted $0. The correct liability is $19,615 of ordinary tax plus $6,133 of preferential-rate tax on $153,455 of taxable income."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It produced a vague estimate built on itemized medical and charity assumptions instead of the $16,100 standard deduction plus the $387 cash charity deduction. $16,811 understates the 24%-bracket ordinary tax plus 15% preferential tax by about $8,900."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It stopped at the $16,100 standard deduction and left out the 2026 non-itemizer cash charitable deduction of $387. Taxable income should be $153,455, not $153,843, which lowers tax by about $93."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It correctly named the non-itemizer charitable deduction, but its $25,611 implies about $957 of deductions beyond the standard deduction. Only the $387 of cash gifts qualifies, so the correct taxable income of $153,455 gives $25,748."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied only the $16,100 standard deduction and left out the $387 non-itemizer cash charitable deduction available in 2026. That overstated ordinary taxable income by $387 and tax by about $93."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It left out the $387 above-the-standard-deduction cash charitable deduction for non-itemizers. That kept taxable income at $153,843 instead of $153,455."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It excluded the $6,889 ESI premiums from wages, which cut AGI to $163,054. It also claimed a $9,000 qualified overtime deduction even though no overtime compensation input exists. The correct calculation uses AGI of $169,943 and only the $16,100 standard plus $387 charity deductions."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the standard deduction but left out the $387 non-itemizer cash charitable deduction. Taxable income stayed $387 too high, and tax came out $93 above the correct figure."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It claimed an $8,400 phased qualified overtime deduction even though no overtime compensation input exists, so the deduction is zero. It also left out the $387 non-itemizer charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It treated the $6,889 employer-sponsored insurance premiums as a pre-tax exclusion from wages, which put AGI at $163,054 instead of $169,943. That understated taxable income by $6,889 and tax by about $1,653."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"$26,859 is $1,112 above the correct figure, which equals 24% of $4,632. That matches failing to exclude the $4,245 traditional 401(k) deferral from wages and also leaving out the $387 non-itemizer charitable deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It guessed $24,500 using itemized deductions and invented nonrefundable credits. The correct calculation uses the $16,100 standard deduction plus $387 cash charity, gives taxable income of $153,455, and has no credits, for $25,748."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied the TCJA sunset: a personal exemption of about $5,300, 2%-floor miscellaneous employee expenses, and pre-TCJA brackets. OBBBA made the TCJA structure permanent, so the correct deductions are the $16,100 standard deduction plus the $387 charity deduction, taxed under the 2026 brackets."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It assumed the TCJA sunset: a $5,300 personal exemption, a $5,331 deduction for 2%-floor employee expenses, and 10/15/25% brackets. Under OBBBA, taxable income is $153,455 after the $16,100 standard deduction and $387 cash charity, taxed under the 2026 brackets."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It used post-TCJA-expiration rules (a $5,450 personal exemption, 2%-floor employee expenses, 10/15/25% brackets) that OBBBA eliminated. The correct calculation uses the $16,100 standard deduction plus the $387 non-itemizer charity deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It applied TCJA-sunset rules (a personal exemption, miscellaneous itemized deductions, 10/15/25% brackets) and deducted the $198 traditional IRA contribution. That deduction is fully phased out for an active 401(k) participant at this MAGI."
+us,scenario_110,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It itemized $21,146, including $5,331 of 2%-floor unreimbursed employee expenses, but OBBBA eliminated that deduction permanently. Without it, itemized deductions fall below the $16,100 standard deduction, which is used together with the $387 cash charity deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for this output, so there is no answer to score."
+us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted an $8,425.50 phased overtime deduction even though no qualified overtime compensation input exists. It also deducted the $198 traditional IRA contribution, which is phased out for an active plan participant at this MAGI."
+us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It claimed the engine returned $0 without computing anything. A single filer with $153,455 of taxable income owes $19,615 of ordinary tax plus $6,133 at 15% on preferential income."
+us,scenario_110,federal_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It used the right structure but left out the $387 non-itemizer charitable deduction. Even without that deduction, $26,360 overshoots the $25,841 that the 2026 brackets ($12,400/$50,400/$105,700) and the $16,100 standard deduction produce, so its brackets or standard deduction were wrong."
+us,scenario_110,federal_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It excluded the $6,889 ESI premiums from wages, which put AGI at $163,054 instead of $169,943. That understated taxable income by $6,889 and tax by about $1,653."
+us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its reasoning computed $26,973, yet it submitted $43,879.20. That computation also skipped the $4,245 traditional 401(k) exclusion and the $18 short-term gain, used guessed brackets, and left out the $387 charity deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It understated AGI at $113,617 when all income sources sum to $169,943. It also took a QBI deduction even though no qualified business income exists, and subtracted invented nonrefundable credits of $194.77."
us,scenario_110,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_110,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its own reasoning says employee Social Security and Medicare on $100,000 of wages come to about $7,650, but it submitted 0 anyway. It skipped the final computation and returned a placeholder zero instead of the $6,200 + $1,450 it had already identified."
us,scenario_110,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It treated the $6,889 employer-sponsored insurance premium as a pre-tax cafeteria-plan deduction and cut FICA wages to $93,111. The household facts list no salary reduction, so FICA applies to the full $100,000 of gross wages, giving $6,200 + $1,450 = $7,650."
@@ -9494,57 +9596,58 @@ us,scenario_110,payroll_tax,muse-wonder,llm_error,payroll_tax_base,False,"It red
us,scenario_110,payroll_tax,qwen-3.7-max,llm_error,other,False,"Its reasoning correctly computed $6,200 Social Security plus $1,450 Medicare = $7,650, but it submitted the unrelated figure $9,351.90. The final value contradicts its own derivation."
us,scenario_110,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It added a $252.55 Additional Medicare Tax even though the only wages are $100,000, well below the $200,000 single-filer threshold. The 0.9% tax applies only to wages and self-employment income, not investment income, so the correct total is $6,200 + $1,450 = $7,650."
us,scenario_110,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"Its own working used the outdated $360.69 base instead of the 2026 $332 base, a $1,700 exemption instead of $1,900, and subtracted the phased-out $198 IRA deduction, which gave about $4,266. It then submitted $5,052, which contradicts its own calculation; the correct figure is $332 + 2.75% × ($168,042.94 − $26,050) = $4,236.81."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It got Ohio taxable income right at $168,043 and the 2.75% rate above $26,050, but left out the $332 fixed base in the 2026 schedule. That makes it $331.81 short ($3,905 vs $4,236.81)."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted the $15,000 federal standard deduction, which Ohio does not have, instead of Ohio's $1,900 personal exemption. It also deducted the phased-out $198 IRA contribution and applied vague 'progressive' brackets instead of $332 + 2.75% of income over $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It never calculated anything and submitted $0, even though $168,042.94 of Ohio taxable income produces $332 + 2.75% × $141,992.94 = $4,236.81."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It applied the repealed 3.5% bracket above $103,500 and got about $4,328. It then dropped that result for an unexplained $2,900. The 2026 schedule is a flat 2.75% above $26,050 plus a $332 base."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied 3.5% to all income over $26,050 with a $360.69 base, getting $5,325, and then submitted an unexplained $4,517. The 2026 rate is 2.75% above $26,050 with a $332 base, applied to taxable income of $168,042.94 (not $167,895, which came from deducting the phased-out IRA contribution)."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It subtracted tax-exempt interest that was never in AGI and used a roughly $2,500 exemption instead of $1,900. Its $3,900 matches 2.75% of the excess over $26,050 with the $332 base left out."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It left out the $332 base in the 2026 Ohio schedule. It also deducted the $198 IRA contribution, which is fully phased out for someone covered by a workplace 401(k) at this income, so its AGI is $169,745 instead of $169,943."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It taxed income over $26,050 at 3.5% instead of the 2026 rate of 2.75% (with a $332 base). It also subtracted $10,915 of federal itemized deductions, which Ohio does not allow, instead of the $1,900 exemption, and took the $20 exemption credit even though income is above that credit's $30,000 limit."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It made up a bracket of about 3.99% above $115,300 and a retirement income credit, then reported about $3,200 without showing math. The 2026 schedule is $332 + 2.75% above $26,050 on $168,042.94, with no credits."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It wrongly said no personal exemption is allowed at this income; Ohio allows $1,900 above $80,000. It also left out the $332 base, taxing 2.75% of $143,893 instead of computing $332 + 2.75% × $141,992.94."
-us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used outdated 2.765%/3.302%/3.688% graduated rates and treated the exemption as fully phased out. The correct calculation is the 2026 schedule of $332 + 2.75% above $26,050, applied after a $1,900 exemption."
-us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied 2.75% to all income, ignoring the $26,050 zero bracket and the $332 base, and dropped the $1,900 exemption. It also started from an understated $163,054 AGI (the $6,889 employer health premiums wrongly subtracted) and added the $954 tax-exempt interest, which is not part of Ohio AGI here."
-us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It applied a 3.5% bracket above $100,000 that does not exist in the 2026 schedule and left out the $332 base. Its AGI of $162,856 wrongly subtracts the $6,889 employer health premiums and the phased-out $198 IRA contribution."
-us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used the $2,400 lowest-income exemption instead of the $1,900 that applies above $80,000. It also applied made-up graduated brackets ($551 + $1,612 + $2,827) instead of $332 + 2.75% of income over $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied the repealed 3.5% bracket above $100,000, left out the $332 base, and used a $2,500 exemption instead of $1,900. The correct figure is $332 + 2.75% × ($168,042.94 − $26,050)."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It subtracted 'standard deductions', which Ohio does not have, and reported $2,981 without any derivation. Ohio subtracts only the $1,900 exemption, leaving $168,042.94 taxed at $332 + 2.75% above $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It started from an AGI of $163,054, which wrongly removes the $6,889 of employer health premiums from wages; they are already excluded from the stated wages. It also used progressive brackets instead of the 2026 schedule of $332 + 2.75% above $26,050 on $168,042.94, and ended up $4 off only because these errors offset each other."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It assumed brackets topping out at 3.5% and gave $3,850 with no exemption or base shown. That figure is about 2.75% of the excess over $26,050 with the $332 base left out. The correct figure is $332 + 2.75% × $141,992.94 = $4,236.81."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It reported $3,280 without showing any work, about $957 below the correct $332 + 2.75% × ($168,042.94 − $26,050). That means it did not apply the 2026 rate schedule to taxable income of $169,942.94 AGI less the $1,900 exemption."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It showed no derivation, and its $4,240.52 is $3.71 above the exact result. That means it did not apply the exact $332 base and 2.75% rate to the exact taxable income of $168,042.94 ($169,942.94 AGI less the $1,900 exemption)."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"It gave $3,986 with no work shown, $251 below $332 + 2.75% × ($168,042.94 − $26,050). That means its taxable income was understated or it left out part of the 2026 schedule."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It wrongly denied the personal exemption above $80,000; Ohio allows $1,900 at that income. It also left out the $332 base, computing 2.75% × ($169,943 − $26,050) instead of $332 + 2.75% × ($168,042.94 − $26,050)."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"Its taxable income of $168,043 is correct, but it used the old $360.69 base instead of the 2026 base of $332. That overstates the tax by $28.69 ($4,265.50 vs $4,236.81)."
-us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It treated the Ohio exemption as fully phased out; $1,900 still applies above $80,000. It also taxed income above $100,000 at the repealed 3.5% rate and left out the $332 base, instead of $332 + 2.75% of all income over $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It applied a 3.5% bracket above $100,000, which the 2026 schedule does not have, and dropped the $1,900 personal exemption. The 2026 tax is $332 + 2.75% × ($168,042.94 − $26,050)."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,missing_output,False,"It acknowledged the tax is positive but declined to calculate it and submitted $0. The correct $4,236.81 follows from $168,042.94 of taxable income under the $332 + 2.75% schedule."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It subtracted federal itemized deductions (mortgage interest, charity), which Ohio does not allow, and its $5,625 implies roughly a 3.5% rate on nearly all income. Ohio subtracts only the $1,900 exemption and charges $332 + 2.75% above $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It got Ohio taxable income right at $168,043 but applied only 2.75% to the amount over $26,050, leaving out the $332 base. That makes it $331.81 short."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its $4,418 matches applying the repealed 3.5% bracket to income over $100,000 on taxable income of about $168,043. The 2026 schedule taxes everything over $26,050 at 2.75% plus a $332 base."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It applied 2.75% above the zero bracket after the exemption but left out the $332 base. That makes it $331.81 short of $4,236.81."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used a $1,850 exemption instead of the $1,900 for income over $80,000 and left out the $332 base. So it computed 2.75% of the excess alone instead of $332 + 2.75% × $141,992.94."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used the right $332 + 2.75% schedule and the $1,900 exemption. But its AGI of $163,054 wrongly subtracts the $6,889 of employer health premiums, and it added the $954 tax-exempt interest, leaving taxable income at $162,108 instead of $168,042.94."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It applied a 3.5% bracket above $100,000 that the 2026 schedule does not have and skipped the $1,900 exemption. The correct tax is $332 + 2.75% of the $141,992.94 over $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It skipped the $1,900 personal exemption and left out the $332 base. So it computed 2.75% × ($169,943 − $26,050) instead of $332 + 2.75% × ($168,042.94 − $26,050)."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It applied the correct $332 + 2.75% schedule and the $1,900 exemption, but started from an AGI of $163,054. That AGI wrongly subtracts the $6,889 of employer health premiums, which are already excluded from the stated wages. The correct Ohio AGI is $169,942.94."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It gave $3,999.05 with no work shown, $237.76 below $332 + 2.75% × ($168,042.94 − $26,050). That means it did not combine the $332 base, the 2.75% rate, and the $1,900 exemption on $169,942.94 of AGI."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It asserted $3,200 with no derivation and cited 'modest credits' that do not apply at this income. The correct figure is $332 + 2.75% × ($168,042.94 − $26,050) = $4,236.81, with no nonrefundable credits."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"Its taxable income of $168,043 is correct, but it used the 2024 brackets with 3.5% above $100,000. The 2026 schedule is a flat 2.75% above $26,050 plus a $332 base."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"Its taxable income of $168,043 is correct, but it applied 2.75% only to the excess over $26,050 and left out the $332 base. That makes it $331.81 short."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It added the $954 tax-exempt interest to Ohio income, which is not part of Ohio AGI here. It also taxed income over $100,000 at the repealed 3.5% rate instead of $332 + 2.75% above $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It deducted the phased-out $198 IRA contribution and used the $2,400 exemption instead of $1,900. It also left out the $332 base in the 2026 schedule."
-us,scenario_110,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It used the pre-2026 three-tier schedule ($2,394 on the first $100,000 plus 3.5% above) and an exemption that was too large. The 2026 tax is $332 + 2.75% of everything over $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"No answer or explanation was returned for this output, so there was no value to score."
-us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It left out the $332 base and subtracted a $20 exemption credit, which applies through $30,000 of Ohio modified AGI less personal exemptions. It also deducted the phased-out $198 IRA contribution, putting AGI at $169,745."
-us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It made up a PolicyEngine result of $0 instead of calculating. $168,042.94 of Ohio taxable income produces $332 + 2.75% × $141,992.94 = $4,236.81."
-us,scenario_110,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It applied a 3.5% rate above $100,000 that was repealed for 2026, left out the $1,900 exemption, and skipped the $332 base. The correct calculation is $332 + 2.75% × ($168,042.94 − $26,050)."
-us,scenario_110,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"Its taxable income of $161,154 starts from an AGI that wrongly subtracts the $6,889 of employer health premiums. It also left out the $332 base, computing 2.75% × $135,104 instead of $332 + 2.75% × $141,992.94."
-us,scenario_110,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used a $1,700 exemption instead of the $1,900 allowed above $80,000 and left out the $332 base, applying only 2.75% to the excess over $26,050."
-us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It added the $954 tax-exempt interest to AGI, made up 2.765%/3.225% brackets, and applied a nonexistent $250 credit plus an arbitrary further cut. The 2026 schedule is $332 + 2.75% above $26,050 on $168,042.94, with no credits."
-us,scenario_110,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It reduced Ohio taxable income to about $96,517 with deductions that do not exist in Ohio. Ohio subtracts only the $1,900 exemption from $169,942.94 of AGI, leaving $168,042.94 taxed at $332 + 2.75% above $26,050."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"It used the outdated $360.69 base instead of the 2026 $332 base, deducted the phased-out $198 IRA contribution, and used a $1,700 exemption instead of $1,900. It then submitted $5,052, which contradicts its own $4,265.55 calculation. The correct result is $332 + 2.75% × ($168,042.94 − $26,050) = $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It got the correct taxable income of $168,043 but taxed only 2.75% of the amount over $26,050. It left out the $332 base in the 2026 Ohio schedule: $3,904.81 + $332 = $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It used the correct $332 + 2.75% schedule and the $1,900 exemption, but subtracted the $198 traditional IRA contribution from AGI. As an active 401(k) participant with MAGI near $170,000, the head gets no IRA deduction, so Ohio AGI is $169,943, not $169,745. That accounts for the $5.45 shortfall."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted a $15,000 federal standard deduction, but Ohio has no standard deduction and allows only the $1,900 personal exemption at this income. It also deducted the phased-out IRA contribution and never applied the $332 + 2.75%-over-$26,050 schedule; its $3,645 comes from unspecified brackets."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It submitted a $0 placeholder without any calculation, even though Ohio taxable income of $168,042.94 produces $332 + 2.75% × $141,992.94 = $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"It used an outdated schedule with inflation-indexed thresholds and a 3.5% bracket above $103,500, reaching about $4,328. It then replaced that with $2,900 without any arithmetic. The 2026 schedule is a single rate: $332 + 2.75% of taxable income above $26,050, applied to $168,042.94."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It applied 3.5% to all income above $26,050 on top of the old $360.69 base, getting $5,325, and then submitted an unsupported $4,517. The 2026 schedule is $332 + 2.75% above $26,050. It also deducted the phased-out IRA contribution and used a $1,850 exemption instead of $1,900."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It subtracted the $954 tax-exempt interest, which was never in AGI, used an exemption of about $2,500 instead of $1,900, and assumed a 3.5% upper bracket that no longer exists in 2026. Its $3,900 total also leaves out the $332 base in the $332 + 2.75%-over-$26,050 schedule."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It left out the $332 base and taxed only 2.75% of income over $26,050. It also subtracted the $198 traditional IRA contribution, which is fully phased out for a 401(k) participant at this income, so it used AGI of $169,745 instead of $169,943."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It applied a flat 3.5% rate instead of the 2026 rate of 2.75% plus a $332 base. It also subtracted $10,915 in itemized deductions, which Ohio does not allow; only the $1,900 personal exemption applies. Finally, it claimed the $20 exemption credit, which is limited to filers with Ohio income below $30,000."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It made up brackets of 3.5% above $100,000 and about 3.99% above $115,300, subtracted unspecified credits such as a retirement income credit, and submitted a $3,200 guess. The 2026 tax is $332 + 2.75% × ($168,042.94 − $26,050), with no nonrefundable credits."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It skipped the $1,900 personal exemption, which still applies above $80,000 of OAGI, and left out the $332 base. As a result it computed only 2.75% × $143,893 instead of $332 + 2.75% × $141,992.94."
+us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It treated the exemption as fully phased out instead of $1,900 and applied an outdated multi-bracket schedule (2.765%, 3.302%, 3.688%) instead of the 2026 schedule of $332 + 2.75% above $26,050. It also deducted the phased-out IRA contribution."
+us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied 2.75% to the entire income, with no $26,050 zero bracket, no $332 base, and no $1,900 exemption. It started from AGI of $163,054 by subtracting the $6,889 employer-sponsored insurance premiums, which are not in reported wages and do not reduce AGI. It also added back the $954 tax-exempt interest, which does not enter Ohio AGI here."
+us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It started from AGI of $162,856 by subtracting both the $6,889 employer insurance premiums and the phased-out $198 IRA contribution, then added back $954 of tax-exempt interest. It then applied the old 3.5% bracket above $100,000 without the $332 base. The 2026 schedule is $332 + 2.75% above $26,050, applied to $168,042.94."
+us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used the $2,400 lowest-income exemption instead of the $1,900 exemption for OAGI above $80,000. It then applied a made-up three-tier rate stack totaling $4,990.36 instead of $332 + 2.75% of income over $26,050."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used a $2,500 exemption instead of $1,900 and applied the old 3.5% bracket above $100,000 without the $332 base. For 2026, Ohio taxes all income over $26,050 at 2.75% plus $332."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It subtracted standard deductions, which Ohio does not have, and gave $2,981 with no arithmetic. The correct path is $169,942.94 OAGI less the $1,900 exemption, then $332 + 2.75% of income over $26,050 = $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It started from AGI of $163,054 by subtracting the $6,889 employer-sponsored insurance premiums, skipped the $1,900 exemption, and applied the old 2.75%/3.5% brackets without the $332 base. Its $4,241 ($2,033.63 + 3.5% × $63,054) lands near $4,236.81 only because these errors offset each other."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It cited an outdated 3.5% top rate and gave $3,850 without arithmetic. The 2026 schedule of $332 + 2.75% over $26,050, applied to $168,042.94, gives $4,236.81, so its figure is too low."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave $3,280 with no calculation. Applying the $1,900 exemption to $169,942.94 OAGI and the $332 + 2.75%-over-$26,050 schedule gives $4,236.81, so its figure understates the tax by about $957."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $4,240.52 equals 2.75% × $73,950 + 3.5% × $63,054. That means it used AGI of $163,054 (subtracting the $6,889 employer insurance premiums), skipped the $1,900 exemption, and applied the old 3.5% bracket without the $332 base. It ended up near the correct value only because these errors offset each other."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"It gave $3,986 with no calculation. That figure matches taxing about $171,000 at 2.75% above $26,050 with no personal exemption and no $332 base, whereas the 2026 schedule adds $332 to 2.75% of $141,992.94."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,thresholds_rates,False,"It claimed no personal exemption is allowed above $80,000, but Ohio allows $1,900 there. It also left out the $332 base and computed only 2.75% × $143,893 = $3,957.06."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"It reached the correct taxable income of $168,043 but used the old $360.69 base instead of the 2026 $332 base. That gives $360.69 + $3,904.81 = $4,265.50 instead of $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It treated the personal exemption as fully phased out, but Ohio allows $1,900 above $80,000 OAGI. It also applied the old 3.5% bracket above $100,000 without the $332 base. For 2026, Ohio taxes all income over $26,050 at 2.75% plus $332."
+us,scenario_110,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It applied the old 2.75%/3.5% split at $100,000 to the full $169,943 with no exemption and no $332 base. The 2026 tax is $332 + 2.75% × ($168,042.94 − $26,050), with the $1,900 exemption already subtracted."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It submitted $0 even though it acknowledged the tax is positive, skipping the straightforward calculation of $332 + 2.75% × ($168,042.94 − $26,050) = $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It subtracted federal itemized deductions (mortgage interest and charitable gifts), which Ohio does not allow, and gave an unsupported $5,625. Ohio subtracts only the $1,900 personal exemption and applies $332 + 2.75% above $26,050."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It got the correct taxable income of $168,043 but taxed only 2.75% of the amount above $26,050. It left out the $332 base in the 2026 Ohio schedule, which brings the total to $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"It gave $4,418 without arithmetic. That figure matches the old 2.75%/3.5% split at $100,000 rather than the 2026 schedule of a single 2.75% rate above $26,050 plus the $332 base."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It applied 2.75% above $26,050 to the correct taxable income after the exemption but left out the $332 base. That gives about $3,905 instead of $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used a $1,850 exemption instead of $1,900 and left out the $332 base, taxing only 2.75% of income over $26,050."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used the correct $332 + 2.75% schedule and $1,900 exemption, but started from AGI of $163,054 by subtracting the $6,889 employer-sponsored insurance premiums, which are not in reported wages. It also added back $954 of tax-exempt interest. Ohio AGI is $169,942.94."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,thresholds_rates,False,"It applied the old 2.75%/3.5% brackets split at $100,000 to $169,943, skipping both the $1,900 exemption and the $332 base. The 2026 schedule taxes all income over $26,050 at 2.75% plus $332."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It skipped the $1,900 personal exemption and left out the $332 base, computing only 2.75% × ($169,943 − $26,050) = $3,957.06."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It used the correct $332 + 2.75% schedule and $1,900 exemption, but started from AGI of $163,054 by subtracting the $6,889 employer-sponsored insurance premiums, which are not in reported wages. Ohio AGI is $169,942.94."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It gave $3,999.05 with no arithmetic. Applying the $1,900 exemption to $169,942.94 OAGI and the $332 + 2.75%-over-$26,050 schedule gives $4,236.81, so it understated the taxable base or left out the $332 base."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"It gave a $3,200 guess with no calculation and invented unspecified credits. The correct path is $168,042.94 taxable income under the $332 + 2.75%-over-$26,050 schedule, with no nonrefundable credits, giving $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It got the correct taxable income of $168,043 but applied the 2024 schedule's 3.5% bracket above $100,000 without the $332 base. For 2026, Ohio taxes all income over $26,050 at 2.75% plus $332."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It got the correct taxable income of $168,043 but computed only 2.75% × $141,993 = $3,904.81. It left out the $332 base in the 2026 Ohio schedule."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-4.7,llm_error,thresholds_rates,False,"It added back $954 of tax-exempt interest and applied the old 3.5% bracket above $100,000 without the $332 base. The 2026 schedule is $332 + 2.75% above $26,050, applied to $168,042.94."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It left out the $332 base, used the $2,400 exemption instead of the $1,900 that applies above $80,000 of OAGI, and deducted the phased-out $198 IRA contribution, using AGI of $169,745."
+us,scenario_110,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied the old 3.5% bracket to income above $100,000. For 2026, Ohio taxes all income over $26,050 at a single 2.75% rate plus a $332 base, which gives $4,236.81 on $168,042.94."
+us,scenario_110,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for this output, so there was no answer to score."
+us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It left out the $332 base and claimed the $20 exemption credit, which is limited to filers with Ohio income below $30,000. It also deducted the phased-out $198 IRA contribution, using AGI of $169,745."
+us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It falsely claimed PolicyEngine returned $0 and did no calculation. Ohio taxable income of $168,042.94 produces $332 + 2.75% × $141,992.94 = $4,236.81."
+us,scenario_110,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It applied the old 2.75%/3.5% split at $100,000 to roughly $170,000 of income. The 2026 schedule taxes everything above $26,050 at 2.75% plus a $332 base, after the $1,900 exemption."
+us,scenario_110,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It started from AGI of about $163,054 by subtracting the $6,889 employer insurance premiums, giving taxable income of $161,154. It also left out the $332 base, taxing only 2.75% of $135,104."
+us,scenario_110,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It used a $1,700 exemption instead of the $1,900 that applies above $80,000 of OAGI and left out the $332 base, computing only 2.75% × $142,193."
+us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It included the $954 tax-exempt interest and the full $100,000 of wages without excluding the $4,245 401(k) deferral. It then used made-up 2.765%/3.225% brackets, a $250 credit, and an arbitrary further reduction to $3,148.93, instead of $332 + 2.75% of income over $26,050."
+us,scenario_110,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It cut Ohio taxable income to about $96,517 with large standard and other deductions that Ohio does not allow. The only subtraction is the $1,900 personal exemption, which leaves $168,042.94 to be taxed at $332 + 2.75% above $26,050."
us,scenario_110,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_111,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated about $16,600 of Social Security as taxable even though its own stated combined income of $30,940 is below the $32,000 joint-filer threshold, not above the $44,000 upper threshold. It then calculated zero taxable income after the standard deduction but replaced the resulting $0 tax with an unsupported $1,362 estimate."
us,scenario_111,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $5,405.50 of Social Security in AGI, but still correctly found that the standard deduction reduced taxable income and tax liability to $0. It then submitted $1,400 despite deriving no taxable income and identifying no tax or credit computation that produces that amount."
@@ -9616,108 +9719,110 @@ us,scenario_112,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,
us,scenario_112,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model correctly stated that passive real-estate farm rent is not subject to self-employment tax, then contradicted that rule by calculating tax as though the entire $1,920 were farm-operator self-employment earnings. The actual net self-employment tax base is zero because the household has no positive net self-employment earnings."
us,scenario_112,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model improperly combined farm rent, financial assistance, and the capital loss into a self-employment tax base. Financial assistance and capital gains or losses are not self-employment earnings, and the household’s relevant net earnings are nonpositive, so no self-employment tax applies."
us,scenario_112,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated the full $1,920 of farm rent as net self-employment income and applied the standard 92.35% and 15.3% factors without establishing positive net earnings from self-employment. The resulting tax base is zero, and its submitted $269 also does not match its own stated calculation of about $271."
-us,scenario_112,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"Counted the $160/month farm rent as SNAP income, which gave net income of about $1,429 and failed the 100% FPL net test. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Included $160/month of farm rent in countable income, which pushed net income to about $1,429, above the $1,305 net limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent and found gross income of $1,737/month, failing the gross test. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"Never worked out income, deductions, or eligibility tests, and defaulted to $0 on vague uncertainty. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"Added the $1,920 of farm rent to gross income ($1,737/month) and failed the household on a roughly $1,650 130% FPL gross limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"Treated the $13,000 of financial assistance as a non-countable gift, but PolicyEngine counts it as $1,083.33/month of unearned income. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Left the $13,000 of financial assistance out of countable income and used only wages plus farm rent, which produced a near-maximum benefit of about $275/month. PolicyEngine counts both the financial assistance and farm rent as unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Assumed the $13,000 of financial assistance is not countable, which gave net income of $346 and a benefit of $194/month. PolicyEngine counts both the financial assistance and farm rent as unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent, giving gross income of $20,843 a year, and failed the household on a $19,584 130% FPL gross limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,Added farm rent to wages and financial assistance and failed the household on the 130% FPL gross test. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted.
-us,scenario_112,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Excluded the $13,000 of financial assistance from countable income, which gave net income of $346 and a benefit of $194/month. PolicyEngine counts both the financial assistance and farm rent as unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"Included $1,920 of farm rent, giving gross income of $20,843, which failed the roughly $20,345 130% FPL limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Used gross income of $1,737/month, which includes farm rent, and failed the household against a $1,672 130% FPL limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Concluded that because 30% of net income exceeds the one-person maximum, the benefit is $0. The current engine counts farm rent and rounds the held FY2026 minimum allotment to $24 per month for an eligible one-person household. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"Reported an annual net income of $6,454 that matches no consistent deduction sequence, then compared 30% of that annual figure ($1,936) with a monthly maximum allotment to reach $0. For an eligible one-person household, the held FY2026 minimum allotment is $24 per month after rounding. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Computed net income of $1,440.20 by including $160/month of farm rent, and failed the household on the net income test. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Gave no computation. The policyengine-us 2.60.0 engine counts farm rent, and Texas BBCE does not apply the net income test to this household. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"Treated '30% of net income exceeds the maximum allotment' as a $0 benefit. For an eligible one-person household, the held FY2026 minimum allotment is $24 per month after rounding. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"Included farm rent to reach net income of $1,440, then set the benefit to $0 because 30% of net exceeds the maximum. The current engine counts farm rent and rounds the held FY2026 minimum allotment to $24 per month for an eligible one-person household. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Gave no reasoning. The policyengine-us 2.60.0 engine counts farm rent, and Texas BBCE does not apply the net income test to this household. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Equated 30% of net income exceeding the maximum allotment with a $0 benefit. For an eligible one-person household, the held FY2026 minimum allotment is $24 per month after rounding. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"Found that total income, farm rent included, exceeds the net income limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Mixed vehicle and asset resources into the net income calculation. Texas BBCE applies a $5,000 asset limit and excludes this vehicle, so the household's $10 bank balance passes the asset test. It also treated 30% of net income exceeding the maximum as $0, skipping the minimum allotment that an eligible one-person household receives. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gemini-4-argon,prompt_ambiguity,categorical_eligibility,False,"Counted only wages and dropped the $1,083.33/month of financial assistance, which gave net income of $185.87 and a near-maximum benefit. PolicyEngine counts both the financial assistance and farm rent as unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Included farm rent to reach net income of about $1,434, above the 100% FPL net limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"Used a roughly $17,000 annual 130% FPL gross limit, well below the actual one-person limit of about $1,695/month. The $11,838 vehicle is excluded from Texas's $5,000 BBCE asset test, which the $10 bank balance passes. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Asserted that income and resources are too high without any computation. Texas BBCE applies a $5,000 asset limit and excludes this vehicle, so the household's $10 bank balance passes the asset test. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Declined to compute and returned $0. Weekly work hours were unlisted, and the $287.68 annual reference assumed 40 hours a week. This output is excluded from scoring."
-us,scenario_112,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent in gross income ($20,843 a year) and failed the household on the gross income limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,Failed the household on the gross income limit after adding farm rent to wages and financial assistance. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted.
-us,scenario_112,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,Counted farm rent and so failed the household on the net income test. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted.
-us,scenario_112,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,Included farm rent and failed the household on the gross income limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted.
-us,scenario_112,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent, so income after the earned-income and standard deductions exceeded the net income limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"Included farm rent, which put net income above the SNAP net limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent along with wages and financial assistance, concluding income was too high for any benefit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,Included farm rent in countable income and concluded no benefit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted.
-us,scenario_112,snap,gpt-6.1-terra,prompt_ambiguity,categorical_eligibility,False,Counted farm rent with wages and financial assistance and concluded income was too high. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted.
-us,scenario_112,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"Asserted that assets exceed SNAP limits. Texas BBCE applies a $5,000 asset limit and excludes this vehicle, so the household's $10 bank balance passes the asset test. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,Counted farm rent and found that gross or net income exceeded the Texas limits. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted.
-us,scenario_112,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,"Concluded that because 30% of net income exceeds the one-person maximum, the formula gives $0. The current engine counts farm rent and rounds the held FY2026 minimum allotment to $24 per month for an eligible one-person household. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Summed wages, farm rent, and financial assistance to $20,843, so net income exceeded the 100% FPL limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Used gross income of $20,843 including farm rent, which failed a roughly $20,700 130% FPL limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,inkling,prompt_ambiguity,categorical_eligibility,False,Counted farm rent and failed the household on the 130% FPL gross test. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted.
-us,scenario_112,snap,kimi-k2.6,prompt_ambiguity,categorical_eligibility,False,"Counted the $1,920 of farm rent toward a $20,843 gross income and applied the 130% FPL gross test. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent and concluded that net income eliminated the benefit. It also denied TANF-based categorical eligibility, although Texas extends BBCE through TANF non-cash assistance. The policyengine-us 2.60.0 engine counts farm rent, and Texas BBCE does not apply the net income test to this household. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Claimed a single adult with wage income does not qualify for SNAP, but wages do not disqualify anyone, and no income test was ever applied. For an eligible one-person household, the held FY2026 minimum allotment is $24 per month after rounding. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,muse-spark-1.3,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent to reach $20,843 and failed the household on the 130% FPL gross test. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,muse-wonder,prompt_ambiguity,categorical_eligibility,False,"Used countable income of $1,737/month including farm rent, so net income exceeded the one-person net limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Computed gross income of $1,737/month including farm rent and failed the household against a $1,695 130% FPL limit. The policyengine-us 2.60.0 engine counts farm rent as SNAP unearned income. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"Its own calculation, counting financial assistance, reached a negative formula benefit. It then reported $3,576, which is the full maximum allotment for 12 months, by invoking an unlisted shelter deduction. For an eligible one-person household, the held FY2026 minimum allotment is $24 per month after rounding. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_112,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Declared the household ineligible without any calculation. The policyengine-us 2.60.0 engine counts farm rent, and Texas BBCE does not apply the net income test to this household. The $287.68 annual reference is frozen and excluded from scoring because weekly work hours are unlisted."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized total of $19,518 included no state income tax in SALT, counted OTC costs the engine excludes, and skipped the 0.5% charity floor. The frozen reference includes an unlisted Virginia withholding estimate and an imputed Part B premium, giving taxable income of $72,807 compared with the model's $78,077. It then dropped its own bracket result of about $11,889 and submitted $8,971, a number its reasoning does not support."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It used its own $3,524 Virginia income tax in SALT and counted OTC costs the engine excludes. The frozen reference uses an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Part B premium. Its itemized deductions were $22,510 compared with the reference's $24,787, giving taxable income of about $75,085 compared with the reference's $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $19,829 of taxable Social Security out of income and used a made-up $28,300 standard deduction. It also omitted the senior deduction, added employee Medicare payroll tax to income tax, and then tacked on an unexplained $2,000 'safe harbor' adjustment. Its $10,268 is not a real tax computation."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed deductions bring taxable income to zero, ignoring AGI of about $106,573. The frozen reference has itemized, senior, and QBI deductions totaling $33,765, taxable income of $72,807 and tax of $10,729.61; these figures depend on unlisted withholding and Part B inputs."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It entered SALT at the $10,000 cap even though the only listed tax was $4,064 of real estate tax. It also applied the full $6,000 senior deduction without the 6% phase-down above $75,000 MAGI. It then threw out its own $10,711 result and submitted an unsupported $5,489."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It made up about $4,500 of mortgage interest even though none was listed. It left out the OBBBA senior deduction ($4,105.65 after the phase-down) and the Virginia income tax component of SALT. It then cut its own $11,802 result to an arbitrary $11,210."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted half of a self-employment tax that does not exist, since the partnership income carries no SE tax here. It took the standard deduction, which the recorded audit also gives without the unlisted withholding and Part B inputs. The frozen reference includes those inputs and itemizes $24,787. Finally it replaced its own ~$11,900 estimate with an unsupported $6,900."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT figure of $7,618.50 uses its own Virginia tax, while the frozen reference uses an unlisted $5,367.30 withholding estimate, for SALT of $9,431.59. Its medical deduction counted OTC costs the engine excludes and omitted the $2,434.80 Part B premium the engine imputes. Its itemized deductions were $2,248 below the reference's, putting taxable income at $75,055 compared with the reference's $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It imputed $6,693 of mortgage interest that was never listed and included no state income tax in SALT. The frozen reference uses an unlisted Virginia withholding estimate and an imputed Part B premium. It counted OTC costs the engine excludes, skipped the 0.5% charity floor, and omitted the OBBBA senior deduction. It also used estimated pre-OBBBA brackets instead of the $12,400/$50,400 breakpoints."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction and itemized $19,518, giving taxable income of $82,183. The frozen reference includes an unlisted Virginia withholding estimate and an imputed Part B premium, giving taxable income of $72,807. It also used low bracket breakpoints ($12,150/$49,400) instead of the 2026 values of $12,400/$50,400."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $21,188 of itemized deductions was $3,599 below the frozen reference's $24,787. The reference includes an unlisted Virginia withholding estimate and an imputed $2,434.80 Part B premium. Its taxable income was $76,407 compared with the reference's $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted an estimated $1,721 half-SE-tax deduction from AGI, although the engine has no self-employment tax for these inputs, and omitted the senior deduction. Its itemized deductions included no state income tax or Part B premium; the frozen reference supplies both as unlisted inputs. Its taxable income was $82,357 compared with the reference's $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the OBBBA senior deduction ($4,105.65). Its medical deduction of $2,007 excludes OTC costs, which the engine also excludes. The frozen reference's $4,441.86 medical deduction includes an imputed Part B premium. Its SALT uses its own Virginia tax, while the reference uses an unlisted withholding estimate. Its taxable income was $80,631 compared with the reference's $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA law (a personal exemption with 15%/25% brackets) even though OBBBA made the TCJA brackets permanent for 2026. It also dropped the QBI deduction and the senior deduction, which together reduce taxable income by about $8,978."
-us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"With the reference's AGI and QBI deduction and no senior deduction, its $79,373 taxable income implies itemized deductions of about $22,328. Its explanation gives no deduction components. The frozen reference itemizes $24,787 using unlisted withholding and Part B inputs and also takes a $4,105.65 senior deduction."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA 2026 rules: a $5,000 personal exemption with 10%/15%/25% brackets. Under OBBBA, 2026 uses the 10%/12%/22% brackets and a $4,105.65 senior deduction. It also subtracted half of a nonexistent SE tax from AGI."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It reported gross income of $110,094 before unspecified adjustments and itemized deductions; its explanation gives no senior or QBI deduction amounts. Its taxable income of about $81,300 exceeds the frozen reference's $72,807, which depends on unlisted withholding and Part B inputs."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the expired pre-TCJA regime (a personal exemption with higher bracket rates) instead of the OBBBA 2026 law. It therefore missed the 10%/12%/22% brackets, the QBI deduction, and the $4,105.65 senior deduction."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used the TCJA 'sunset' brackets and a $5,050 personal exemption, which do not exist in 2026 under OBBBA. It omitted the senior deduction and itemized $19,256, below the frozen reference's $24,787, which includes unlisted withholding and Part B inputs."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. Under the 2026 brackets, $9,334 corresponds to taxable income of about $66,460; with the reference's AGI, this implies deductions roughly $6,300 above the frozen reference's $33,765."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the OBBBA senior deduction ($4,105.65) and itemized $22,018, giving taxable income of $79,682. The frozen reference itemizes $24,787 using unlisted withholding and Part B inputs, giving taxable income of $72,807. Its bracket arithmetic also overstated tax on that income."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It says it took the standard deduction with the additional elderly amount and the QBI deduction and names no senior deduction; it gives no deduction amounts or taxable income. A reconstruction using the reference's AGI and QBI deduction, the $18,150 aged standard deduction for 2026 and no other deductions gives taxable income near $83,550. The frozen reference takes the $4,105.65 senior deduction and itemizes $24,787 using an unlisted Virginia withholding estimate in SALT and an imputed Part B premium in medical expenses."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized total of $17,518 counted only real estate tax in SALT and medical expenses without the Part B premium. The frozen reference includes an unlisted Virginia withholding estimate and an imputed Part B premium. It also omitted the $4,872 QBI deduction and the $4,105.65 senior deduction, giving taxable income above the reference's $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 law with a personal exemption and 15%/25% brackets. OBBBA made the 10%/12%/22% brackets permanent and added the phased $4,105.65 senior deduction alongside the QBI deduction."
-us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired. That led it to eliminate the QBI deduction, add a $5,500 personal exemption, and tax income at 15%/25% rates. Under OBBBA, 2026 keeps QBI and the 12%/22% brackets and adds the $4,105.65 senior deduction."
-us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction and itemized $17,518, giving taxable income of $84,183. The frozen reference includes an unlisted Virginia withholding estimate and an imputed $2,434.80 Part B premium, giving taxable income of $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard deduction offsets all taxable income, ignoring AGI of $106,573. The frozen reference's deductions total $33,765 and leave $72,807 taxable, producing $10,729.61 of tax; these figures depend on unlisted withholding and Part B inputs."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the filer as having modest income mostly from Social Security. Wages, partnership income, the pension, the IRA, interest, the capital loss and 85% of benefits give AGI of about $106,573. The frozen reference's taxable income of $72,807 and tax of $10,729.61 depend on unlisted withholding and Part B inputs."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the post-2025 pre-TCJA regime with a personal exemption and the higher bracket structure. It omitted the QBI deduction and the OBBBA senior deduction. Its itemized deductions of $17,518 were below the frozen reference's $24,787, which includes unlisted withholding and Part B inputs."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted half of a nonexistent self-employment tax from AGI and took the standard deduction. Its taxable income of about $67,020 was about $5,800 below the frozen reference's $72,807, which depends on unlisted withholding and Part B inputs."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It chose the $18,150 standard deduction plus the $1,000 non-itemizer charitable deduction. Its $11,970 answer matches the recorded $11,969.80 alternative without withholding, the local sales-tax estimate or a Part B premium. The frozen reference includes unlisted withholding and Part B inputs, itemizes $24,787 and reaches taxable income of $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Under the 2026 brackets, its $11,540 corresponds to taxable income of about $76,490, roughly $3,700 above the frozen reference's $72,807. Its explanation gives no deduction amounts establishing which components caused the difference; the reference includes unlisted withholding and Part B inputs."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized deductions of $20,639 and $11,642.20 answer match the recorded $20,639.28 and $11,642.16 alternative using Virginia liability in SALT without a Part B premium. The frozen reference includes an unlisted Virginia withholding estimate and an imputed Part B premium, giving taxable income of $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its explanation lists itemized and QBI deductions but gives no deduction amounts. With the reference's AGI and QBI deduction, and assuming no senior deduction, its $80,675 taxable income would imply itemized deductions of about $21,026. The frozen reference takes a $4,105.65 senior deduction and itemizes $24,787 using unlisted withholding and Part B inputs, reaching taxable income of $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It stated AGI of $96,572.80, about $10,000 below the engine's $106,572.56, which also inflated the senior deduction to $4,705.63. It took the standard deduction, while the frozen reference itemizes $24,787 using unlisted withholding and Part B inputs."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its $11,642.09 answer matches the recorded $11,642.16 alternative using Virginia liability in SALT without a Part B premium. Its explanation gives no itemized amount or breakdown. The frozen reference includes unlisted withholding and Part B inputs and reaches taxable income of $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It used the standard deduction plus the $1,000 non-itemizer charitable deduction, totaling $19,150. Its $11,969.85 answer matches the recorded $11,969.80 alternative without withholding, the local sales-tax estimate or a Part B premium. The frozen reference includes unlisted withholding and Part B inputs, itemizes $24,787 and reaches taxable income of $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. Under the 2026 brackets, $10,234 corresponds to taxable income of about $70,555; with the reference's AGI, this implies deductions roughly $2,250 above the frozen reference's $33,765."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied restored pre-TCJA law (a $5,468 personal exemption with 10%/15%/25% brackets) and no QBI or senior deduction. OBBBA keeps the 12%/22% brackets and the QBI deduction for 2026 and adds the senior deduction."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA sunset: no QBI deduction, a $5,300 personal exemption, and 15%/25% brackets. Under OBBBA, 2026 law includes the $4,872 QBI deduction, the $4,105.65 senior deduction, and the 10%/12%/22% brackets."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 standard deduction without the $1,000 non-itemizer charitable deduction. The frozen reference itemizes $24,787 using unlisted withholding and Part B inputs. Its stated tax of $12,279 exceeds the approximately $12,190 bracket tax on its stated taxable income of $79,445."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 senior deduction and used its own $2,746 Virginia tax in SALT, while the frozen reference uses an unlisted withholding estimate. It counted OTC costs the engine excludes and omitted the Part B premium the engine imputes. It also skipped the 0.5% charity floor, giving taxable income of $79,437 compared with the reference's $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA 2026 law (a personal exemption with higher brackets) and subtracted half of a nonexistent SE tax from AGI. It missed the OBBBA brackets, the QBI deduction, and the senior deduction."
-us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value and no explanation for federal_income_tax_before_refundable_credits, so there is no answer to score."
-us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It compared the standard deduction with an itemized total of $17,518 and took the standard deduction plus the non-itemizer charitable deduction. Its $11,969.85 answer matches the recorded $11,969.80 alternative without withholding, the local sales-tax estimate or a Part B premium. The frozen reference includes unlisted withholding and Part B inputs, itemizes $24,787 and reaches taxable income of $72,807."
-us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It found itemized deductions exceeding the standard deduction but then submitted $0 without a supporting credit computation. The frozen reference's $72,807 taxable income depends on unlisted withholding and Part B inputs. It also imputed mortgage interest that was never listed."
-us,scenario_114,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"It omitted the $4,105.65 phased senior deduction and had itemized deductions of $23,318, giving taxable income of $78,382. The frozen reference's $24,787 itemized deductions and $72,807 taxable income depend on unlisted withholding and Part B inputs. Its $12,175 also overstates the bracket tax on its own taxable income."
-us,scenario_114,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"With the reference's AGI, QBI and senior deductions, its $78,077 taxable income implies itemized deductions of about $19,518. Its explanation gives no itemized breakdown establishing the components. The frozen reference's $24,787 includes unlisted withholding and Part B inputs."
-us,scenario_114,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It took the standard deduction, skipping the $1,000 non-itemizer charitable deduction, because it put itemized deductions at about $19,500. The frozen reference itemizes $24,787 using an unlisted Virginia withholding estimate and an imputed $2,434.80 Medicare Part B premium. The model's standard-deduction computation gives taxable income of $79,445."
-us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $19,829 of taxable Social Security out of income and used an outdated $15,750 standard deduction. It also omitted the senior deduction and itemizing. Its $15,654 is not the bracket tax on its own $66,122 taxable income, which would be about $9,000."
-us,scenario_114,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It understated AGI at $80,164 instead of $106,573 and used the wrong standard deduction and QBI amounts. It also double-counted the QBI deduction by subtracting its tax effect again after already removing it from taxable income."
+us,scenario_112,snap,claude-fable-5,prompt_ambiguity,categorical_eligibility,False,"Counted the $1,920 farm rent ($160/mo) as SNAP income, so net income came to ~$1,429, above the ~$1,305 net limit, and the household failed. PolicyEngine counts only wages plus financial assistance ($1,576.90 gross), giving net income of $1,269.19 (97% FPL). The household is therefore eligible and gets the $23.84/mo minimum allotment."
+us,scenario_112,snap,claude-fable-5.1,prompt_ambiguity,categorical_eligibility,False,"Included farm rent in gross income ($1,737/mo), which put net income at ~$1,429 and over the 100% FPL limit. PolicyEngine's SNAP income leaves out farm rent, so net income is $1,269.19 (97% FPL). That passes the net test and triggers the $23.84 minimum allotment even though 30% of net exceeds the $298 maximum."
+us,scenario_112,snap,claude-fable-5.2,prompt_ambiguity,categorical_eligibility,False,"Assumed the head fails the SNAP work requirement because no work hours were listed, so it never computed income. PolicyEngine treats the work requirement as met for this wage earner and finds the household eligible through the income tests and TX BBCE. It pays the $23.84/mo minimum allotment ($287.68/yr)."
+us,scenario_112,snap,claude-haiku-4.5,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent, giving gross income of $1,737/mo, and ruled the household out on the 130% FPL gross test without considering Texas BBCE at 165%. PolicyEngine's gross income excludes farm rent ($1,576.90, 121% FPL), so the household passes, and its $1,269.19 net income yields the $23.84 minimum allotment."
+us,scenario_112,snap,claude-haiku-5.5,prompt_ambiguity,categorical_eligibility,False,"Declined to establish eligibility and did no income test or benefit calculation. The household is eligible: gross is 121% FPL, net is 97% FPL, assets are $10, and TX BBCE applies. It receives the $23.84/mo minimum allotment, which the $0 answer leaves out."
+us,scenario_112,snap,claude-opus-4.7,prompt_ambiguity,categorical_eligibility,False,"Added farm rent to gross income ($1,737/mo) and failed the household on the ~130% FPL gross test, ignoring Texas BBCE. PolicyEngine's SNAP gross income is $1,576.90 without farm rent (121% FPL), which passes, so the household receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,claude-opus-4.8,prompt_ambiguity,categorical_eligibility,False,"Treated the $13,000 of financial assistance as a non-countable gift. PolicyEngine counts it as unearned income ($1,083.33/mo), which brings net income to $1,269.19. 30% of that ($380.70) exceeds the $298 maximum, so only the $23.84 minimum allotment is paid, not ~$192/mo."
+us,scenario_112,snap,claude-opus-5,prompt_ambiguity,categorical_eligibility,False,"Left the $13,000 of financial assistance out of countable income and estimated ~$275/mo from wages plus farm rent alone. PolicyEngine counts the financial assistance, so net income is $1,269.19 and the 30% contribution exceeds the $298 maximum, leaving only the $23.84 minimum allotment."
+us,scenario_112,snap,claude-opus-5.5,prompt_ambiguity,categorical_eligibility,False,"Assumed the financial assistance does not count and computed $298 − 30% × $346 ≈ $194/mo. PolicyEngine includes the $1,083.33/mo of financial assistance as unearned income, so net income is $1,269.19 and the benefit falls to the $23.84 minimum allotment."
+us,scenario_112,snap,claude-sonnet-4.6,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent with wages and financial assistance ($20,843/yr) and failed the household on a 130% FPL gross test, without applying Texas BBCE. PolicyEngine's SNAP income excludes farm rent ($1,576.90/mo, 121% FPL), so the household passes and gets the $23.84/mo minimum allotment."
+us,scenario_112,snap,claude-sonnet-5,prompt_ambiguity,categorical_eligibility,False,"Concluded that wages, farm rent and financial assistance together exceed the 130% FPL gross limit. PolicyEngine counts only wages and financial assistance ($1,576.90/mo, 121% FPL), so the gross and net tests are met and the one-person household receives the $23.84 minimum allotment."
+us,scenario_112,snap,claude-sonnet-5.5,prompt_ambiguity,categorical_eligibility,False,"Excluded the $13,000 of financial assistance from countable income and arrived at ~$194/mo. PolicyEngine counts it as unearned income, so net income is $1,269.19 and 30% of it exceeds the $298 maximum. Only the $23.84/mo minimum allotment is paid."
+us,scenario_112,snap,deepseek-v4-flash-0731,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent, making gross income $20,843, which exceeds the 130% FPL limit of ~$20,345. It also never considered Texas BBCE. PolicyEngine's SNAP gross income excludes farm rent ($18,923/yr, 121% FPL), so the household is eligible for the $23.84/mo minimum allotment."
+us,scenario_112,snap,deepseek-v4-pro,prompt_ambiguity,categorical_eligibility,False,"Failed the household because gross income of $1,737/mo, including farm rent, exceeded 130% FPL. PolicyEngine's SNAP gross income is $1,576.90 without farm rent (121% FPL), so the household passes and receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,deepseek-v4-pro-0813,prompt_ambiguity,categorical_eligibility,False,"Stopped at $0 because 30% of net income exceeds the one-person maximum. It never applied the rule that eligible one- and two-person households get a minimum allotment (8% of the maximum, $23.84/mo). PolicyEngine finds the household eligible with $1,269.19 net income, so that minimum is paid."
+us,scenario_112,snap,deepseek-v4.1-flash,prompt_ambiguity,categorical_eligibility,False,"Its $6,454 net income comes from leaving out financial assistance and subtracting a single month's $204 standard deduction from annual income. It then compared the annual 30% contribution ($1,936) to a monthly maximum allotment. PolicyEngine's monthly calculation (net $1,269.19) gives the $23.84/mo minimum allotment."
+us,scenario_112,snap,gemini-3-flash-preview,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent and got net income of $1,440, above the net limit, so the household failed the net test required under Texas ECE. PolicyEngine's SNAP income excludes farm rent, so net income is $1,269.19 (97% FPL). That passes, and the household receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,categorical_eligibility,False,"Gave no computation and called the household ineligible. The household meets the gross test (121% FPL), the net test (97% FPL), the asset test and TX BBCE, and receives the $23.84/mo one-person minimum allotment ($287.68/yr)."
+us,scenario_112,snap,gemini-3.1-pro-preview,prompt_ambiguity,categorical_eligibility,False,"Took 30% of net income exceeding the maximum allotment to mean a $0 benefit. Eligible one-person households receive the minimum allotment ($23.84/mo) when the formula goes to zero, and this household passes the eligibility tests with net income of $1,269.19."
+us,scenario_112,snap,gemini-3.5-flash,prompt_ambiguity,categorical_eligibility,False,"Set the benefit to $0 because 30% of net income ($432) exceeds the maximum allotment, without applying the minimum benefit for eligible one-person households. PolicyEngine finds the household eligible (net $1,269.19, without farm rent) and pays the $23.84/mo minimum."
+us,scenario_112,snap,gemini-3.5-flash-lite,prompt_ambiguity,categorical_eligibility,False,"Gave $0 with no computation. The household is eligible: gross income is 121% FPL, net income is 97% FPL, assets are $10 and TX BBCE applies. Because 30% of net exceeds the $298 maximum, it receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,gemini-3.6-flash,prompt_ambiguity,categorical_eligibility,False,"Applied the max-allotment-minus-30%-of-net formula and stopped at $0. Eligible one- and two-person households receive the minimum allotment, which for this household is $23.84/mo ($287.68/yr)."
+us,scenario_112,snap,gemini-3.7-flash,prompt_ambiguity,categorical_eligibility,False,"Counted total income, including farm rent, against the net income limit. PolicyEngine's SNAP income excludes farm rent, so net income is $1,269.19, under the $1,304.17 poverty line, and the household receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,gemini-3.8-flash,prompt_ambiguity,categorical_eligibility,False,"Returned $0 because 30% of net income exceeds the single-person maximum, and also pointed to vehicle and asset resources, which TX BBCE and the $10 bank balance make irrelevant. It never applied the $23.84/mo minimum allotment that eligible one-person households receive."
+us,scenario_112,snap,gemini-4-argon,prompt_ambiguity,categorical_eligibility,False,"Counted only wages and dropped the $13,000 of financial assistance, giving net income of $185.87/mo and a large benefit. PolicyEngine counts the financial assistance as unearned income, so net income is $1,269.19 and the benefit is the $23.84/mo minimum allotment."
+us,scenario_112,snap,glm-5.2,prompt_ambiguity,categorical_eligibility,False,"Included farm rent in gross income, which put net income at ~$1,434 and over 100% FPL, so the household failed the net test. PolicyEngine's SNAP income omits farm rent, so net income is $1,269.19 (97% FPL), the net test is met, and the $23.84/mo minimum allotment is paid."
+us,scenario_112,snap,glm-5.3,prompt_ambiguity,categorical_eligibility,False,"Understated the 130% FPL gross limit as ~$17,000 (it is ~$20,345 for one person). It also said the $11,838 vehicle breaks the asset limit, but TX BBCE removes that test and the countable assets are $10. With the household's income of $18,923/yr, PolicyEngine finds it eligible for the $23.84/mo minimum allotment."
+us,scenario_112,snap,gpt-5.4-mini,prompt_ambiguity,categorical_eligibility,False,"Said income and resources are too high without computing anything. The household passes the gross test (121% FPL), the net test (97% FPL), the $10 asset test and TX BBCE, and receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,gpt-5.4-nano,prompt_ambiguity,categorical_eligibility,False,"Defaulted to $0, saying the inputs could not support a calculation, even though a one-person household's wages, financial assistance and assets were all given. The SNAP tests are met (net income of $1,269.19, 97% FPL) and the $23.84/mo minimum allotment applies."
+us,scenario_112,snap,gpt-5.5,prompt_ambiguity,categorical_eligibility,False,"Counted $20,843 of gross income, including farm rent, and failed the household on the gross limit. PolicyEngine's SNAP gross income leaves out farm rent ($18,923/yr, 121% FPL), so the household passes and gets the $23.84/mo minimum allotment."
+us,scenario_112,snap,gpt-5.6-luna,prompt_ambiguity,categorical_eligibility,False,"Concluded that countable income, including farm rent, exceeds the one-person gross limit. PolicyEngine's gross income excludes farm rent ($1,576.90/mo, 121% FPL), so the household is eligible and receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,gpt-5.6-sol,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent and so failed the household on the net-income test. Without farm rent, PolicyEngine's net income is $1,269.19, below the $1,304.17 poverty line, so the household passes and receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,gpt-5.6-terra,prompt_ambiguity,categorical_eligibility,False,"Included farm rent income and failed the household on the gross-income limit. PolicyEngine's SNAP gross income is wages plus financial assistance only ($1,576.90/mo, 121% FPL), which passes, so the $23.84/mo minimum allotment is paid."
+us,scenario_112,snap,gpt-6-astra,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent and so found net income above the one-person net limit. PolicyEngine's net income excludes farm rent ($1,269.19, 97% FPL), so the household is eligible and gets the $23.84/mo minimum allotment."
+us,scenario_112,snap,gpt-6-luna,prompt_ambiguity,categorical_eligibility,False,"Placed net income above the SNAP limit because farm rent was counted. PolicyEngine computes net income of $1,269.19 without farm rent, which is under 100% FPL, so the household qualifies and receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,gpt-6-sol,prompt_ambiguity,categorical_eligibility,False,"Concluded that income is too high for a positive one-person benefit and ignored the minimum allotment. An eligible one-person household (here net income is $1,269.19, 97% FPL) receives $23.84/mo even when 30% of net income exceeds the $298 maximum."
+us,scenario_112,snap,gpt-6.1-sol,prompt_ambiguity,categorical_eligibility,False,"Treated income after deductions as too high to produce any benefit and left out the minimum allotment rule. The household passes the eligibility tests (net income $1,269.19) and so receives the $23.84/mo one-person minimum."
+us,scenario_112,snap,gpt-6.1-terra,prompt_ambiguity,categorical_eligibility,False,"Called income too high for a SNAP benefit without running the eligibility tests or applying the minimum allotment. The household meets the gross and net tests and TX BBCE, so it receives the $23.84/mo minimum ($287.68/yr)."
+us,scenario_112,snap,grok-4.3,prompt_ambiguity,categorical_eligibility,False,"Gave a generic statement that income and assets exceed limits, with no computation. Countable assets are $10, gross income is 121% FPL and net income is 97% FPL, so the household qualifies for the $23.84/mo minimum allotment."
+us,scenario_112,snap,grok-4.5,prompt_ambiguity,categorical_eligibility,False,"Counted all income, including farm rent, and held that the gross or net limit is exceeded. PolicyEngine's SNAP income excludes farm rent, giving gross of $1,576.90 (121% FPL) and net of $1,269.19 (97% FPL). Both tests pass and the $23.84/mo minimum allotment is paid."
+us,scenario_112,snap,grok-4.6,prompt_ambiguity,categorical_eligibility,False,Applied the 30%-of-net formula and stopped at $0 when the contribution exceeded the one-person maximum. It never applied the $23.84/mo minimum allotment that an eligible one-person household receives.
+us,scenario_112,snap,grok-4.7,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent, so net income stayed above the 100% FPL net limit. PolicyEngine's SNAP income excludes farm rent, so net income is $1,269.19 (97% FPL), which passes and yields the $23.84/mo minimum allotment."
+us,scenario_112,snap,grok-build-0.1,prompt_ambiguity,categorical_eligibility,False,"Compared $20,843 of gross income, including farm rent, with a ~$20,700 limit at 130% FPL and found the household ineligible. PolicyEngine's gross income excludes farm rent ($18,923/yr), which is under the limit, so the household receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,inkling,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent and failed the household on the 130% FPL gross test. PolicyEngine's SNAP gross income without farm rent is $1,576.90/mo (121% FPL), so the household is eligible and gets the $23.84/mo minimum allotment."
+us,scenario_112,snap,kimi-k2.6,prompt_ambiguity,categorical_eligibility,False,"Counted $20,843 of income, including farm rent, against the 130% FPL gross limit and did not apply Texas BBCE. PolicyEngine's gross income excludes farm rent (121% FPL), so the household passes and receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,kimi-k3,prompt_ambiguity,categorical_eligibility,False,Said the household lacks categorical eligibility and that the expected contribution eliminates the allotment. That misses Texas BBCE through TANF non-cash assistance and the minimum benefit for eligible one-person households. PolicyEngine pays $23.84/mo.
+us,scenario_112,snap,minimax-m3,prompt_ambiguity,categorical_eligibility,False,"Treated having wage income as a single person as disqualifying for SNAP, which is not a SNAP rule. The household passes the income tests (net income $1,269.19, 97% FPL) and receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,muse-spark-1.3,prompt_ambiguity,categorical_eligibility,False,"Counted $20,843 of gross income, including farm rent, and failed the 130% FPL test. PolicyEngine's SNAP gross income excludes farm rent ($18,923/yr, 121% FPL), so the household qualifies for the $23.84/mo minimum allotment."
+us,scenario_112,snap,muse-wonder,prompt_ambiguity,categorical_eligibility,False,"Counted farm rent in its $1,737/mo of countable income and found net income above the one-person limit. Without farm rent, PolicyEngine's net income is $1,269.19 (97% FPL), so the net test passes and the $23.84/mo minimum allotment applies."
+us,scenario_112,snap,ox-alpha,prompt_ambiguity,categorical_eligibility,False,"Compared $1,737/mo of gross income, including farm rent, with the ~$1,695 limit at 130% FPL. PolicyEngine's gross income excludes farm rent ($1,576.90/mo), which is under the limit, so the household is eligible and receives the $23.84/mo minimum allotment."
+us,scenario_112,snap,qwen-3.7-max,prompt_ambiguity,categorical_eligibility,False,"Its own calculation showed a negative formula benefit, but it then reported $3,576, roughly 12 × the $298 maximum, with no basis. The correct path is the minimum allotment for an eligible one-person household: $23.84/mo, or $287.68/yr."
+us,scenario_112,snap,qwen3.8-max,prompt_ambiguity,categorical_eligibility,False,"Declared the household ineligible without any computation. It meets the gross test (121% FPL), the net test (97% FPL), the asset test and TX BBCE, and receives the $23.84/mo one-person minimum allotment."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized total of $19,518 leaves Virginia income tax out of SALT, skips the 0.5%-of-AGI charitable floor, and counts the $2,000 of OTC costs instead of the Part B premium, which gave $78,077 of taxable income and about $11,889 of tax. It then cut the answer to $8,971 with no stated basis."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only $3,524 of Virginia income tax in SALT instead of the $5,367.30 withholding estimate. Its medical deduction includes the $2,000 of OTC costs instead of the $2,434.80 Part B premium. Together these understate itemized deductions by about $2,277, so taxable income comes out at $75,085 instead of $72,807."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT deduction is only the $4,064 of real estate tax, with no Virginia income tax (the reference uses $5,367.30 withheld). Its medical deduction includes the $2,000 of OTC costs and leaves out the $2,434.80 Part B premium. This raised taxable income to $78,610."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $19,829 of taxable Social Security out of income and used an invented $28,300 standard deduction even though its own itemized total was higher. It also applied no QBI or senior deduction to taxable income, added Medicare payroll tax to income tax, and then added an arbitrary $2,000."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed deductions wipe out the tax, but AGI is $106,573. Even after $33,765 of deductions, taxable income is $72,807, which is taxed into the 22% bracket for $10,729.61."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It entered SALT at the $10,000 cap instead of the actual taxes, applied the full $6,000 senior deduction without the 6% phase-down above $75,000 of MAGI, and reached $10,711 of tax. It then submitted $5,489, a figure that none of its steps supports."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It made up $4,500 of mortgage interest that the prompt never lists, left Virginia income tax out of SALT, and skipped the charitable floor. It also never applied the phased senior deduction of $4,105.65. It then marked its own $11,802 down to $11,210 for no stated reason."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the partnership income as self-employment income and subtracted half of an SE tax, chose the standard deduction plus an unphased $6,000 senior deduction instead of itemizing, and reached about $11,900. It then dropped the answer to $6,900 with no supporting calculation."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its SALT of $7,618.50 includes only about $3,555 of Virginia income tax instead of the $5,367.30 withholding estimate. Its medical deduction counts the $2,000 of OTC costs instead of the $2,434.80 Part B premium. That leaves itemized deductions $2,248 short and taxable income at $75,055 instead of $72,807."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It made up $6,693 of mortgage interest from the loan balance, left Virginia income tax out of SALT, skipped the charitable floor, and never applied the OBBBA senior deduction. It also used guessed brackets from inflating 2025 figures instead of the 2026 schedule."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $4,105.65 senior deduction and Virginia income tax in SALT, skipped the charitable floor, and counted the OTC costs instead of the Part B premium. This gave $82,183 of taxable income instead of $72,807."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized total of $21,188 is about $3,600 short of $24,787. It used too little Virginia income tax compared with the $5,367.30 withholding estimate and left out the $2,434.80 Part B premium from medical expenses. Taxable income therefore came to $76,407."
+us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted half of a self-employment tax on partnership income, which carries no SE tax here. It also left Virginia income tax out of SALT, did not add the Part B premium to medical expenses, and never applied the $4,105.65 senior deduction, so taxable income reached $82,357."
+us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It never applied the $4,105.65 senior deduction, which is allowed alongside itemizing. It also understated SALT with about $3,552 of Virginia tax instead of $5,367, and its medical deduction leaves out the Part B premium, so taxable income was $80,631."
+us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA sunset law, with a personal exemption and 15%/25% brackets. OBBBA kept the TCJA brackets, removed the personal exemption and kept the QBI deduction for 2026, so it also left out the $4,872 QBI deduction and the $4,106 senior deduction."
+us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $79,373 of taxable income reflects only itemized and QBI deductions. It never applied the $4,105.65 OBBBA senior deduction and understated itemized deductions, mainly the Virginia income-tax SALT component and the Part B premium."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA law with a $5,000 personal exemption and 10/15/25% brackets, although OBBBA kept the TCJA rates for 2026. It also subtracted half of an SE tax on partnership income and left out the QBI and senior deductions."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated gross income at $110,094 instead of $106,573 of AGI and never applied the $4,105.65 senior deduction. It also reported $13,919 of tax, which is more than the 2026 brackets produce on its own $81,300 of taxable income."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 sunset law with personal exemptions and higher brackets. OBBBA kept the TCJA brackets and added the senior deduction, and it also left out the QBI deduction."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the TCJA-sunset brackets and a $5,050 personal exemption, neither of which exists under OBBBA for 2026. It also never applied the $4,105.65 senior deduction, and its $19,256 itemized total leaves out the Virginia income tax and the Part B premium."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. $9,334 matches about $66,460 of taxable income, roughly $6,350 below the correct $72,807, which means deductions were overstated beyond the $24,787 itemized, $4,872 QBI and $4,106 senior amounts."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted only itemized deductions of $22,018 and QBI, and never applied the $4,105.65 OBBBA senior deduction that stacks on itemizing. Its itemized total is also short because it undercounts Virginia income tax and leaves out the Part B premium."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 standard deduction and QBI only, which gives about $83,550 of taxable income. It never itemized, even though itemized deductions of $24,787 are higher, and it left out the $4,105.65 senior deduction."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted only $17,518 of itemized deductions, which leave out Virginia income tax, the Part B premium and the charitable floor. It never applied the $4,872 QBI deduction or the $4,106 senior deduction, which pushed taxable income to about $89,000."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,It applied 2026 sunset rules with a personal exemption and pre-TCJA brackets instead of OBBBA's permanent TCJA brackets. It also left out the QBI and senior deductions.
+us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It said the QBI deduction no longer exists in 2026 and applied a personal exemption with 15%/25% brackets. OBBBA made the TCJA rates and §199A permanent, so it lost both the $4,872 QBI deduction and the $4,106 senior deduction and taxed income at the old rates."
+us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It never applied the $4,105.65 senior deduction. Its $17,518 itemized total leaves out Virginia income tax and the Part B premium and skips the 0.5% charitable floor, so taxable income came to $84,183."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the age-adjusted standard deduction wipes out all taxable income, but AGI is $106,573. Even after $33,765 of deductions, $72,807 is still taxed, giving $10,729.61."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly treated the filer as having modest income mostly from Social Security. Wages, partnership income, the pension, the IRA and 85% of Social Security make AGI $106,573 and leave $72,807 of taxable income."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-2025 sunset brackets and a personal exemption that OBBBA repealed. It also left out the $4,872 QBI deduction and the $4,106 senior deduction, and its $17,518 itemized total leaves out Virginia income tax."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted half of a self-employment tax on partnership income, which has no SE tax here, and took the standard deduction instead of the larger $24,787 itemized total. The result was $67,020 of taxable income, well below $72,807."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It took the $18,150 standard deduction plus the $1,000 non-itemizer charitable deduction, which gives the $11,970 alternative. Itemizing gives $24,787 once SALT includes the $5,367 Virginia withholding estimate and medical includes the $2,434.80 Part B premium, and that saves $1,240."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"$11,540 matches about $76,490 of taxable income, so its itemized deductions came to about $21,100 instead of $24,787. It undercounted the Virginia income tax in SALT and left the Part B premium out of medical expenses."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized total of $20,639 is $4,148 short. It used a Virginia income tax well below the $5,367.30 withholding estimate and left the $2,434.80 Part B premium out of medical expenses, so taxable income came to $76,955."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted only itemized and QBI deductions and never applied the $4,105.65 senior deduction. Its itemized total of about $21,000 is also short because Virginia income tax is undercounted and the Part B premium is missing, so taxable income was $80,675."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It understated AGI by $10,000 ($96,573 instead of $106,573), which also inflated its senior deduction to $4,706. It took the standard deduction instead of the larger $24,787 itemized total."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its itemized deductions came to about $20,639 instead of $24,787. It used too little Virginia income tax compared with the $5,367.30 withholding estimate and left out the Part B premium, so taxable income was about $76,955."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It took the standard deduction plus the $1,000 non-itemizer charitable deduction, which gives the $11,969.85 alternative. It missed that itemized deductions of $24,787, including the Virginia withholding SALT and the Part B premium in medical expenses, are higher and lower the tax by $1,240."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It gave no derivation. $10,234 matches about $70,555 of taxable income, so it overstated total deductions by about $2,250 compared with the correct $33,765 of itemized, QBI and senior deductions."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-TCJA sunset law with a $5,468 personal exemption and 10/15/25% brackets, although OBBBA kept the TCJA brackets. It also left out the QBI and senior deductions."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired, applied a $5,300 personal exemption with 15%/25% brackets, and said QBI was gone. OBBBA kept both the TCJA brackets and §199A and added the senior deduction."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly concluded that the $18,150 standard deduction exceeds itemized deductions, which total $24,787 once SALT includes the $5,367 Virginia withholding estimate and medical includes the Part B premium. On the standard-deduction path it also left out the $1,000 non-itemizer charitable deduction."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It never applied the $4,105.65 senior deduction. Its SALT uses only $2,746 of Virginia tax instead of $5,367, and it leaves out the charitable floor and the Part B premium, so taxable income was $79,437."
+us,scenario_114,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA brackets with a $5,300 personal exemption instead of 2026 OBBBA law and subtracted half of an SE tax on partnership income. It also left out the QBI and senior deductions."
+us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was nothing to score."
+us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It compared an itemized total of $17,518 with the standard deduction. That total leaves out Virginia income tax in SALT and the Part B premium in medical expenses, and skips the charitable floor. The correct itemized figure is $24,787, so it ended on the standard-deduction alternative of $11,969.85."
+us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It recognized that itemizing exceeds the standard deduction but still claimed the tax is close to $0. About $106,573 of AGI less $33,765 of deductions leaves $72,807 taxable, which is $10,729.61 of tax."
+us,scenario_114,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted only itemized deductions of $23,318 and QBI and never applied the $4,105.65 senior deduction, which is allowed alongside itemizing. Taxable income therefore came to $78,382 instead of $72,807."
+us,scenario_114,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"Its $78,077 of taxable income reflects an itemized total of about $19,518. That total leaves Virginia income tax out of SALT, skips the 0.5% charitable floor, and counts the OTC costs instead of the Part B premium."
+us,scenario_114,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It chose the standard deduction because its itemized estimate was about $19,500 and left out Virginia income tax. Correctly itemized deductions are $24,787. It also left out the $1,000 non-itemizer charitable deduction on its own chosen path."
+us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It left the $19,829 of taxable Social Security out of income, used an outdated $15,750 standard deduction, and ignored itemizing and the senior deduction. Its $15,654 of tax is also inconsistent with the brackets applied to its own $66,122 of taxable income."
+us,scenario_114,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It understated AGI at $80,164 instead of $106,573, cut the QBI deduction to $3,550 instead of $4,872, and used the standard deduction instead of itemizing. It then subtracted invented credit amounts to reach $5,290."
us,scenario_114,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model applied the childless EITC phaseout using only $22,700 of wages and ignored the rule requiring the greater of earned income or adjusted gross income. The filer’s other taxable income raises AGI beyond the childless EITC phaseout ceiling, yielding EITC of $0 rather than $295."
us,scenario_114,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, so it failed the required submission contract."
us,scenario_114,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model correctly calculated both employee payroll-tax components and correctly excluded self-employment tax, but submitted $3,470.30 instead of their $1,736.55 sum. Its answer effectively duplicates the employee payroll-tax total despite its own reasoning identifying no additional component."
@@ -9742,57 +9847,58 @@ us,scenario_114,self_employment_tax,inkling,llm_error,payroll_tax_base,False,"Tr
us,scenario_114,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no self_employment_tax value and no explanation, so there is no substantive calculation to assess. The correct output is $0 because the household has no net earnings from self-employment."
us,scenario_114,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"Treated the $24,362 partnership/S-corp income as SE income and computed $3,442.62. It then submitted an unexplained $3,459.40. The pass-through income is not net earnings from self-employment, so SE tax is $0."
us,scenario_114,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"Treated the $24,362 partnership/S-corp income as net SE income, which is the core error because it is not SE earnings, so SE tax is $0. It also added a made-up 'additional Medicare-equivalent portion above the Social Security wage base'. No income here comes near that base, and this pushed its answer to $3,486.63."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Its own steps reached about $3,555, using taxable income of $66,296 with $19,518 itemized and a $930 exemption, but it submitted $2,597 with no basis. That itemized figure counts the $2,000 OTC expenses as medical, leaves out the $2,434.80 Medicare Part B premium that the engine imputes and the prompt does not list and skips the 0.5% charitable floor. It also left out the $800 aged exemption."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It phased out the age deduction from a $75,000 threshold, which is the married threshold, and kept $256. The single threshold is $50,000, so the deduction is $0. Its $18,985 itemized total applied the charitable floor but counted the $2,000 OTC expenses as medical. The engine leaves out OTC spending and instead imputes a $2,434.80 Medicare Part B premium that the prompt does not list."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It built a federal AGI of about $27,752 by subtracting charitable gifts above the line and leaving out much of the income. It then applied a federal-style $28,000 standard deduction. Virginia AGI is actually $86,744, and the engine's $19,420 of itemized deductions, which count an unlisted Part B premium, and $1,730 of exemptions leave $65,594 taxable."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed a retirement income subtraction and the age deduction would wipe out taxable income. Virginia has no general pension or IRA subtraction, and AFAGI of $86,744 fully phases out the age deduction. The engine, which also counts an unlisted Part B premium in itemized deductions, leaves $65,594 taxable."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It used an $8,750 standard deduction and left out the $1,730 of exemptions, which gave $4,227. The engine itemizes $19,420 in Virginia using an unlisted Part B premium. When the engine counts neither that premium nor its Virginia withholding estimate in federal SALT, it also takes the $8,750 standard deduction. It then submitted $1,854, which contradicts its own work."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It computed tax with an $8,500 standard deduction and only the $930 exemption. It then made a vague downward adjustment to $4,014 without computing an itemized total. The engine deducts $19,420 of itemized deductions, including an unlisted Part B premium, and $1,730 of exemptions, leaving $65,594 taxable."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It took an $8,500 standard deduction. In the engine the filer itemizes federally, which gives $19,420 of Virginia itemized deductions, including an unlisted Part B premium. When the engine counts neither that premium nor its Virginia withholding estimate in federal SALT, it also takes the $8,750 Virginia standard deduction. It also treated personal exemptions as a tax credit instead of a $1,730 subtraction from income."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted only the $930 personal exemption and left out the $800 aged exemption the 69-year-old gets. Its $19,518 itemized total also counts the $2,000 OTC expenses as medical, which the engine leaves out in favor of an unlisted $2,434.80 Medicare Part B premium it imputes, and skips the 0.5% charitable floor."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It used a $9,000 standard deduction. The engine itemizes $19,420 with an unlisted Part B premium. When the engine counts neither that premium nor its Virginia withholding estimate in federal SALT, it also takes the standard deduction. It also took the $930 and $800 exemptions straight off the tax as credits instead of deducting $1,730 from taxable income."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It allowed the full $12,000 age deduction, claiming income was under the threshold, but AFAGI of $86,744 is more than $12,000 over the $50,000 single threshold, so the deduction is $0. It also left out the $1,730 of exemptions and the Medicare Part B premium in medical. Then it cut its own $2,947 figure to $2,450 with no basis."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 itemized total leaves out the $2,434.80 Medicare Part B premium in medical and skips the 0.5%-of-AGI charitable floor. The engine applies the floor and imputes the unlisted premium, for a total of $19,420. It also subtracted only the $930 exemption and left out the $800 aged exemption."
-us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It took a roughly $4,800 standard deduction plus an $800 'aged addition' and reached taxable income of about $78,515. The engine deducts $19,420 of Virginia itemized deductions, including an unlisted Part B premium, leaving $65,594 taxable after exemptions. When the engine counts neither that premium nor its Virginia withholding estimate in federal SALT, it also takes the standard deduction."
-us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the full $12,000 age deduction even though AFAGI of $86,744 fully phases it out above the $50,000 single threshold. It used an $8,500 standard deduction, where the engine itemizes $19,420 using an unlisted Part B premium, and left out the $1,730 of exemptions, and these errors largely offset each other."
-us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Its $19,518.04 itemized total counts the $2,000 OTC expenses as medical and skips the 0.5%-of-AGI charitable floor. The engine excludes OTC spending and instead imputes an unlisted $2,434.80 Medicare Part B premium, for a total of $19,419.94, so the model's taxable income is about $98 below the engine's."
-us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It started from federal AGI of $106,573 and never took the $19,829 Virginia subtraction for taxable Social Security. It also used $17,518 itemized, which leaves out the Part B premium and the charitable floor, and only the $930 exemption."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated itemized deductions at $23,948.60 against the engine's $19,419.94, which counts an unlisted Part B premium. It also subtracted only the $800 aged exemption and left out the $930 personal exemption, giving taxable income of $60,294 instead of $65,594."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"There is no usable reasoning. Its $4,180 implies taxable income of about $77,200, which matches subtracting only a standard deduction and one exemption from the $86,744 Virginia AGI, while the engine subtracts $19,420 of itemized deductions, including an unlisted Part B premium, and $1,730 of exemptions. When the engine counts neither that premium nor its Virginia withholding estimate in federal SALT, it also takes the standard deduction."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,726 implies about $17,466 of total deductions from the $86,744 Virginia AGI. That matches roughly the $17,518 itemized figure, which has no Medicare Part B premium and no charitable floor, and it leaves out the $1,730 of exemptions. The engine's total is $21,150, including an unlisted Part B premium."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It understated itemized deductions at $15,511, which is charitable gifts without the 0.5%-of-AGI floor plus real estate taxes, with no medical expense deduction. The engine's $19,420 applies the floor and deducts $4,441.86 of medical expenses, including a $2,434.80 Medicare Part B premium that the engine imputes. It also used a wrong $1,600 exemption instead of $930 personal plus $800 aged, or $1,730."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"There is no reasoning. Its $5,617 implies taxable income of about $102,000, which matches taxing nearly all of federal AGI without the $19,829 Social Security subtraction or the $19,420 itemized deductions."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518.04 itemized total leaves out the $2,434.80 Medicare Part B premium in the medical deduction and skips the 0.5%-of-AGI charitable floor. The engine's total, with the floor and an unlisted premium it imputes, is $19,419.94, so the model's taxable income came out $1,902 above the engine's."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,623.52 matches $17,518 of itemized deductions. That figure leaves out the $2,434.80 Medicare Part B premium in medical and the 0.5%-of-AGI charitable floor, against the engine's $19,419.94, which counts that unlisted premium, so its implied taxable income is $1,902 above the engine's."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 itemized total leaves out the $2,434.80 Medicare Part B premium in medical expenses and skips the 0.5%-of-AGI charitable floor. The engine's total, with the floor and an unlisted premium it imputes, is $19,420, so the model's taxable income came out $67,496 against the engine's $65,594."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518.04 itemized total leaves out the $2,434.80 Medicare Part B premium in the medical deduction and skips the 0.5%-of-AGI charitable floor. The engine's total, with the floor and an unlisted premium it imputes, is $19,419.94, so the model's taxable income was $1,902 above the engine's."
-us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It invoked a Virginia pension deduction that does not exist and a standard deduction. When the engine counts neither its unlisted Part B premium nor its Virginia withholding estimate in federal SALT, it also takes the standard deduction. The model's $14,036.62 is impossible under its own stated brackets for any taxable income under $87,000. The engine's taxable income of $65,594, which reflects itemized deductions with an unlisted Part B premium, gives $3,514."
-us,scenario_114,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It made an arithmetic error ($106,573 − $19,829 shown as $87,744) and took an $8,500 standard deduction, where the engine itemizes $19,420 using an unlisted Part B premium and takes an $8,750 standard deduction when it counts neither that premium nor its Virginia withholding estimate in federal SALT. It also used a $930 aged exemption instead of the $800 amount."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed deductions and exemptions shelter all income. Virginia AGI is $86,744 after the Social Security subtraction, and the engine's $19,420 of itemized deductions, which count an unlisted Part B premium, plus $1,730 of exemptions still leave $65,594 taxable."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed retirement income and the age 65+ status drive taxable income to zero. Virginia taxes the $39,600 of pension and IRA income, and the age deduction is fully phased out at $86,744 AFAGI, and the engine, counting an unlisted Part B premium in itemized deductions, leaves $65,594 taxable."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 itemized total leaves out the $2,434.80 Medicare Part B premium in medical expenses and skips the 0.5%-of-AGI charitable floor. The engine's total, with the floor and an unlisted premium it imputes, is $19,420, so the model's taxable income was $1,902 above the engine's."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the Virginia age deduction even though AFAGI of $86,744 is more than $12,000 over the $50,000 single threshold, so the deduction is $0. That pushed taxable income down to about $51,647 instead of $65,594."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It used an $8,750 standard deduction. When the engine counts neither an unlisted $2,434.80 Part B premium nor a $5,367.30 Virginia withholding estimate in federal SALT, it also takes the federal standard deduction and the $8,750 Virginia standard deduction. With both inputs, it itemizes $19,420 in Virginia. It also subtracted only the $930 exemption and left out the $800 aged exemption."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"There is no usable derivation. Its $4,658 implies taxable income of about $85,500, which matches taking almost nothing off the $86,744 Virginia AGI compared with the engine's deductions of $19,420 itemized, including an unlisted Part B premium, and $1,730 in exemptions."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the 0.5%-of-AGI charitable floor but left out the $2,434.80 Medicare Part B premium, which the prompt does not list and the engine imputes. Its $16,985.18 of itemized deductions matches the engine's $16,985.14 without the premium, and its $3,654.16 is within a cent of the adjudicated $3,654.15 alternative."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,397 implies taxable income of about $63,557, so it overstated itemized deductions by about $2,000. That matches counting the $2,000 OTC health expenses as deductible medical on top of the engine's $19,420."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its about $57,229 of taxable income matches carrying into Virginia the $96,572.80 federal AGI stated in its federal answer, about $10,000 below the engine's: $76,744 of Virginia AGI less about $17,785 of itemized deductions and $1,730 of exemptions. The engine's $19,419.94 itemized total includes an unlisted Part B premium."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the 0.5%-of-AGI charitable floor but left out the $2,434.80 Medicare Part B premium, which the prompt does not list and the engine imputes. Its $16,985.18 of itemized deductions matches the engine's $16,985.14 without the premium. Virginia's rates on its $68,028.82 taxable income give $3,654.16, within a cent of the adjudicated $3,654.15 alternative, and its submitted $3,654.66 adds a $0.50 arithmetic slip."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,654.16 matches $16,985.18 of itemized deductions. That figure applies the charitable floor and leaves out the $2,434.80 Medicare Part B premium, which the prompt does not list and the engine imputes in its $19,419.94. Its answer is within a cent of the adjudicated $3,654.15 alternative."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard deduction and exemptions offset all tax. Virginia AGI is $86,744, and the engine's $19,420 of itemized deductions, which count an unlisted Part B premium, plus $1,730 of exemptions leave $65,594 taxable at up to 5.75%."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 itemized total leaves out the $2,434.80 Medicare Part B premium in medical expenses and skips the 0.5%-of-AGI charitable floor. The engine's total, with the floor and an unlisted premium it imputes, is $19,420, so the model's taxable income was $67,496 against the engine's $65,594."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Its $19,518 itemized total counts the $2,000 OTC expenses as medical and skips the 0.5%-of-AGI charitable floor. The engine excludes OTC spending and instead imputes an unlisted $2,434.80 Medicare Part B premium, for a total of $19,419.94."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the filer takes the federal standard deduction and used an $8,500 Virginia standard deduction. The engine itemizes federally when it counts its unlisted Part B premium or its $5,367.30 Virginia withholding estimate in federal SALT, and with both it deducts $19,420 in Virginia. When it counts neither input, it takes the federal standard deduction and the $8,750 Virginia standard deduction, for $76,263.77 of taxable income."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the full $12,000 age deduction even though $86,744 AFAGI fully phases it out above $50,000. It left out the $1,730 of exemptions and used a 5.3% top rate instead of Virginia's 5.75%."
-us,scenario_114,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted only the $930 personal exemption and left out the $800 aged exemption. Its stated components ($86,744 − $19,647 − $930 = $66,167) do not match the $64,446 taxable income it actually taxed."
-us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value or explanation for state_income_tax_before_refundable_credits, so there was nothing to score."
-us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the filer does not itemize federally and took an $8,500 standard deduction. The engine itemizes $19,420 with its unlisted Part B premium; when it counts neither that premium nor its $5,367.30 Virginia withholding estimate in federal SALT, it also takes the standard deduction, $8,750 in Virginia. It also used a $930 aged exemption instead of $800."
-us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the filer as a low-income retiree with no tax. Virginia AGI is $86,744, and the engine's $19,420 of itemized deductions, which count an unlisted Part B premium, and $1,730 of exemptions leave $65,594 taxable."
-us,scenario_114,state_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"It took a roughly $8,000 standard deduction with the exemptions, leaving taxable income of about $77,014. The engine itemizes $19,420 using an unlisted Part B premium, leaving $65,594 taxable after exemptions. When it counts neither that premium nor its Virginia withholding estimate in federal SALT, it also takes the standard deduction, $8,750."
-us,scenario_114,state_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It used the single standard deduction where the engine itemizes $19,420 using an unlisted Part B premium; when it counts neither that premium nor its Virginia withholding estimate in federal SALT, it also takes the standard deduction. It also subtracted only the $930 personal exemption and left out the $800 aged exemption, giving $77,314 taxable."
-us,scenario_114,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It identified the $19,829 Social Security subtraction but then subtracted deductions from full federal AGI of $106,573. It used a standard deduction with an invented $1,000 aged add-on and left out the $1,730 of exemptions. The engine itemizes $19,420 using an unlisted Part B premium; when it counts neither that premium nor its Virginia withholding estimate in federal SALT, it also takes the standard deduction."
-us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It left taxable Social Security out of AGI but still subtracted the full $23,328 of benefits, and it used an outdated $1,000 standard deduction. It also applied a $12,000 age deduction that is fully phased out. It then submitted $2,453.71, which does not match its own computed $2,542.44."
-us,scenario_114,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It used an understated federal AGI of $80,164, a $9,000 standard deduction (the engine uses $8,750 when it counts neither its unlisted Part B premium nor its Virginia withholding estimate in federal SALT), and the full $12,000 age deduction, which $86,744 AFAGI phases out. It also invented $745.50 of nonrefundable credits."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It used $19,518 of itemized deductions, with no 0.5% charitable floor and with the $2,000 OTC expenses counted as medical instead of the Part B premium. It also applied only the $930 exemption and left out the $800 aged exemption, which got it to $3,555. It then cut that to $2,597 without citing any rule."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It used the $75,000 married-filing-jointly threshold for the age deduction and kept $256 of it. The single threshold is $50,000, so AFAGI of $86,744 removes the deduction entirely. Its $18,985 of itemized deductions also counted the $2,000 OTC expenses as medical instead of the $2,434.80 Medicare Part B premium."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Its medical deduction counted the $2,000 of OTC health expenses, which are not deductible, and left out the $2,434.80 Medicare Part B premium. That put itemized deductions at $18,985.18 instead of $19,419.94, which raised taxable income to $66,028.82 and the tax by $25."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It built federal AGI incorrectly: it left out the $19,829 of taxable Social Security and subtracted charitable gifts and employee expenses above the line. It then applied a federal-style standard deduction and got zero. Under Virginia rules, the Social Security subtraction, $19,419.94 of itemized deductions and $1,730 of exemptions leave $65,593.83 of taxable income."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed an age deduction and a retirement-income subtraction would bring taxable income to zero. Virginia has no general pension subtraction, and the $12,000 age deduction is fully phased out at AFAGI of $86,744. Pension, IRA, wage and partnership income therefore stay taxable."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It used the $8,750 standard deduction instead of the $19,419.94 of Virginia itemized deductions, which apply because the filer itemizes federally. It also applied no personal or aged exemptions and got $4,227, then submitted $1,854, which its own work does not support."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It started from the $8,500 standard deduction and only the $930 exemption, leaving out the $800 aged exemption. It then cut its $4,188 result to $4,014 as an unexplained 'adjustment' instead of computing the $19,419.94 of itemized deductions."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It used the $8,500 standard deduction instead of the $19,419.94 of itemized deductions. It also treated the personal exemptions as a tax credit of about $281, but Virginia's $930 personal and $800 aged exemptions are subtracted from taxable income."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $800 Virginia aged exemption, so exemptions were $930 instead of $1,730. Its $19,518 of itemized deductions also skipped the 0.5% charitable floor and counted OTC expenses instead of the Part B premium, giving $66,296 of taxable income instead of $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It took a $9,000 standard deduction instead of $19,419.94 of itemized deductions. It also subtracted the $930 personal and $800 aged exemptions directly from tax as credits. They are deductions from income and are worth only about 5.75% of $1,730."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the full $12,000 age deduction, but AFAGI of $86,744 is more than $12,000 over the $50,000 single threshold, so the deduction is fully phased out. It also left out the $1,730 of exemptions and then cut its own $2,947 result to $2,450 without explanation."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 of itemized deductions skipped the 0.5%-of-AGI charitable floor and left the $2,434.80 Medicare Part B premium out of medical expenses. It also applied only the $930 exemption without the $800 aged exemption, giving $68,296 of taxable income instead of $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It started from a wrong federal AGI of $104,874. It then applied a standard deduction of about $4,800 plus $800, instead of $19,419.94 of itemized deductions and $1,730 of exemptions, which overstated taxable income at about $78,515."
+us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the full $12,000 age deduction, which is fully phased out at AFAGI of $86,744 for a single filer. It also used the $8,500 standard deduction instead of $19,419.94 of itemized deductions and left out the $1,730 of exemptions. These errors partly cancelled out."
+us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Its $19,518.04 of itemized deductions skipped the 2026 charitable floor of 0.5% of AGI (about $533). It also counted the $2,000 of OTC expenses as medical instead of the $2,434.80 Medicare Part B premium, which overstated deductions by $98 and understated tax by $5.63."
+us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It never subtracted the $19,828.80 of federally taxable Social Security, which Virginia fully excludes, and it left out the $800 aged exemption. That left $88,125 of taxable income instead of $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated itemized deductions at $23,948.60 instead of $19,419.94. It also allowed only an $800 exemption, leaving out the $930 personal exemption from the $1,730 total, and its $60,294.40 taxable income does not match its own inputs."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $4,180 implies about $77,174 of taxable income. That matches Virginia AGI minus a standard deduction and one exemption, skipping the $19,419.94 of itemized deductions that apply because the filer itemizes federally."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,726 implies about $69,280 of taxable income. That matches roughly $17,500 of itemized deductions with no charitable floor and no Part B premium, and no $1,730 of personal and aged exemptions. Correct taxable income is $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the medical deduction entirely; the correct amount is $4,441.86 after the 7.5% floor, including the Part B premium. It also skipped the 0.5% charitable floor and used $1,600 of exemptions instead of $930 + $800 = $1,730."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its $5,617 implies about $102,000 of taxable income. That matches taxing almost all of federal AGI without the $19,828.80 Social Security subtraction or the $19,419.94 of itemized deductions."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 of itemized deductions applied no 0.5%-of-AGI floor to charitable gifts ($11,447 instead of $10,913.78). It also left the $2,434.80 Medicare Part B premium out of medical expenses ($2,007 instead of $4,441.86), understating deductions by $1,902."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,623.52 matches $17,518 of itemized deductions: charitable gifts with no 0.5%-of-AGI floor and medical expenses without the $2,434.80 Medicare Part B premium. That understates the correct $19,419.94 by $1,902."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used $17,518 of itemized deductions, skipping the 2026 0.5%-of-AGI charitable floor and leaving the $2,434.80 Medicare Part B premium out of medical expenses. Correct itemized deductions are $19,419.94, giving $65,593.83 of taxable income."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518.04 of itemized deductions applied no 0.5%-of-AGI charitable floor and left the $2,434.80 Medicare Part B premium out of medical expenses. That understates deductions by $1,902 and overstates tax by $109."
+us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"$14,036.62 does not follow from its own method. Taxable income after the Social Security subtraction and deductions is under $80,000, and at Virginia's 5.75% top rate that produces about $4,000. It also cited a pension deduction for people 65 and older that does not exist and skipped the $19,419.94 of itemized deductions."
+us,scenario_114,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted wrongly to get $87,744 instead of $86,744. It used the $8,500 standard deduction instead of $19,419.94 of itemized deductions and a $930 aged exemption instead of $800. It also computed the tax as $270 plus 5.75% above $8,000, instead of $720 plus 5.75% above $17,000."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed deductions and exemptions shelter all income. Virginia AGI is $86,744 after the Social Security subtraction, and $19,419.94 of itemized deductions plus $1,730 of exemptions leave $65,593.83 taxed at up to 5.75%."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed age and retirement income bring taxable income to near zero. Virginia has no pension exclusion, and the age deduction is fully phased out at AFAGI of $86,744, leaving $65,593.83 taxable."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 of itemized deductions applied no 0.5%-of-AGI floor to charitable gifts. It also left the $2,434.80 Medicare Part B premium out of medical expenses, understating the correct $19,419.94 by $1,902."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the age deduction even though AFAGI of $86,744 is more than $12,000 over the $50,000 single threshold, so the deduction is zero. That brought taxable income down to about $51,647 instead of $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It used an $8,750 standard deduction instead of $19,419.94 of Virginia itemized deductions, which are required because the filer itemizes federally. It also applied only the $930 exemption and left out the $800 aged exemption."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $4,658 implies about $85,500 of taxable income, essentially Virginia AGI with no itemized deductions. It left out the $19,419.94 of itemized deductions and most of the $1,730 of exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $16,985.18 of itemized deductions applies the charitable and medical floors correctly but leaves out the $2,434.80 Medicare Part B premium that PolicyEngine adds to medical expenses. One reading follows the prompt's instruction not to infer unlisted health coverage, so no premium. The other treats a 69-year-old as enrolled in Medicare Part B, so the premium is deductible. That premium accounts for the whole $140 gap."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,397 implies about $63,557 of taxable income, roughly $2,037 below the correct $65,593.83. Its itemized deductions and exemptions together exceed the correct $19,419.94 + $1,730."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its $57,229 of taxable income implies about $27,800 of itemized deductions, roughly $8,400 more than the correct $19,419.94. That matches deducting medical expenses without subtracting the 7.5%-of-AGI floor."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its $16,985.18 of itemized deductions applies the 0.5% charitable floor and the 7.5% medical floor but leaves out the $2,434.80 Medicare Part B premium the engine counts as a medical expense. One reading follows the prompt's ban on inferring health coverage, so no premium. The other treats a 69-year-old as enrolled in Part B. That premium explains the gap."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $3,654.16 matches $16,985.18 of itemized deductions, which apply both floors but leave out the $2,434.80 Medicare Part B premium the engine adds to medical expenses. One reading follows the prompt's 'do not infer health coverage' instruction. The other treats a 69-year-old as enrolled in Part B. That premium is the entire gap."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the standard deduction and exemptions offset all tax. Those total about $10,000, far below Virginia AGI of $86,744, and even $19,419.94 of itemized deductions plus $1,730 of exemptions leave $65,593.83 taxable."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its $17,518 of itemized deductions applied no 0.5%-of-AGI charitable floor. It also left the $2,434.80 Medicare Part B premium out of medical expenses, understating the correct $19,419.94 by $1,902."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Its $19,518 of itemized deductions skipped the 2026 charitable floor of 0.5% of AGI. It also counted the $2,000 of OTC expenses as medical instead of the $2,434.80 Medicare Part B premium, giving $65,496 of taxable income instead of $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the filer takes the federal standard deduction and so used Virginia's $8,500 standard deduction. Federal itemized deductions (charitable, medical and SALT) exceed the federal senior standard deduction, so Virginia requires itemizing at $19,419.94."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It applied the $12,000 age deduction even though it is fully phased out at AFAGI of $86,744. It also left out the $1,730 of exemptions and used a 5.3% top rate instead of Virginia's 5.75%."
+us,scenario_114,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It applied only the $930 personal exemption and left out the $800 aged exemption. It also used about $19,647 of itemized deductions instead of $19,419.94, and its $64,446 of taxable income does not match its own figures, since $86,744 − $19,647 − $930 = $66,167."
+us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no value and no explanation for state_income_tax_before_refundable_credits, so there was no answer to score."
+us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the filer does not itemize federally and used the $8,500 standard deduction instead of $19,419.94 of itemized deductions. It also set the aged exemption at $930 instead of $800."
+us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the filer as a low-income retiree with no tax. Virginia AGI is $86,744 after the Social Security subtraction, and itemized deductions plus $1,730 of exemptions leave $65,593.83 taxable."
+us,scenario_114,state_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"It used an $8,000 standard deduction plus exemptions instead of $19,419.94 of itemized deductions, which overstated taxable income at about $77,014 instead of $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It used the $8,500 standard deduction and only the $930 exemption instead of $19,419.94 of itemized deductions and $1,730 of exemptions, which overstated taxable income at $77,314."
+us,scenario_114,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It said Virginia excludes Social Security but computed taxable income from full federal AGI of $106,573, never subtracting the $19,828.80. It also used a standard deduction with an aged add-on of $1,000 that Virginia does not have, instead of itemized deductions and $1,730 of exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It invented a deduction for half of self-employment tax on partnership income and used a $1,000 standard deduction. It also applied the fully phased-out $12,000 age deduction and subtracted $23,328 of Social Security it had never included in AGI. The $2,453.71 it submitted does not match its own $2,542.44."
+us,scenario_114,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It started from a wrong federal AGI of $80,164 and applied the $12,000 age deduction, which is fully phased out at AFAGI of $86,744. It also used a standard deduction instead of $19,419.94 of itemized deductions and invented $745.50 of nonrefundable credits."
us,scenario_114,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_115,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an Alabama aged/disabled Medicaid pathway for the head instead of determining whether PolicyEngine assigned an eligible category. The engine assigns medicaid_category = NONE, so comparing $18,708 of Social Security income with an asserted aged/disabled limit cannot produce eligibility."
us,scenario_115,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age, disability, low assets, and $1,953 of monthly income as sufficient for an Alabama aged/disabled Medicaid category. PolicyEngine assigns no Medicaid category, so its income-and-assets comparison was applied to a pathway the head does not qualify through."
@@ -9805,56 +9911,57 @@ us,scenario_115,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model
us,scenario_115,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model waived the gross-income test, excluded tax-exempt interest, and converted a negative allotment formula into an unsupported $142 monthly benefit. Total income of approximately $23,442 exceeds the applicable eligibility limit, and minimum-benefit rules cannot override that ineligibility."
us,scenario_115,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly computed that the ordinary allotment formula yields $0 even after excluding tax-exempt interest, but then replaced that result with an unsupported $321 monthly minimum benefit. It also missed that approximately $23,442 of income exceeds the applicable SNAP eligibility threshold; minimum-allotment rules do not grant benefits after failure of eligibility."
us,scenario_115,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced all countable income to zero through a standard medical deduction. The listed $250 of annual medical and over-the-counter expenses cannot erase approximately $23,442 of annual household income, which exceeds the applicable one-person SNAP eligibility limit."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It assumed the $4,734 of federally tax-exempt interest is also exempt in Alabama. Alabama taxes interest on non-Alabama obligations, so Alabama AGI is $4,734. After the $3,000 standard deduction and $1,500 exemption, $234 is taxable at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"It treated tax-exempt interest as untaxed in Alabama. Alabama has no federal muni exclusion, so the $4,734 is Alabama AGI. Subtracting the $4,500 of combined standard deduction and exemption leaves $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It excluded the tax-exempt interest from Alabama income and assumed extra exemptions for the elderly. Alabama includes the $4,734 of interest in AGI and gives no age-based extra deduction. Only $3,000 plus $1,500 applies, leaving $234 taxable and $4.68 of tax."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It assumed tax-exempt interest is excluded and that the deductions for a 77-year-old exceed the income. Alabama taxes the $4,734 of interest, and the $3,000 standard deduction plus $1,500 exemption leave $234 taxable. At 2%, that is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It treated tax-exempt interest as non-taxable in Alabama. Alabama gross income includes interest on other states' obligations, so AGI is $4,734. After $4,500 of deductions and exemption, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It set Alabama AGI to $0 by carrying over the federal exclusion for tax-exempt interest. Alabama adds that $4,734 back. The $3,000 standard deduction and $1,500 exemption leave $234 taxable, for $4.68 at 2%."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It assumed Alabama does not tax tax-exempt interest. Alabama includes interest on non-Alabama obligations, so the $4,734 is Alabama AGI. Subtracting $4,500 leaves $234 taxable at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"It looked only at the Social Security exemption and said there was no other taxable income. It ignored the $4,734 of tax-exempt interest, which Alabama includes in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It stated that tax-exempt interest is not included in Alabama gross income. Alabama taxes interest on obligations of other states, so AGI is $4,734. That leaves $234 taxable after $4,500 of deductions and exemption, for $4.68 of tax."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It treated the $4,734 of tax-exempt interest as not subject to Alabama tax. Alabama does not follow the federal muni exclusion. After the $3,000 standard deduction and $1,500 exemption, $234 is taxable at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"It started from federal AGI of $0 and never added back tax-exempt interest. Alabama adds back the $4,734, which leaves $234 taxable after $4,500 of deductions and exemption. At 2%, that is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It excluded tax-exempt interest and assumed deductions for the elderly. Alabama includes the $4,734 of interest in AGI and gives no age-based deduction. Only the $3,000 standard deduction and $1,500 exemption apply, leaving $234 taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It said tax-exempt interest is not taxed in Alabama. Alabama taxes interest on non-Alabama obligations, so AGI is $4,734 and taxable income is $234 after $4,500 of deductions. At 2%, the tax is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,thresholds_rates,False,"It correctly considered including the interest, but it used a $4,500 standard deduction for filers 65 and older plus a $1,500 exemption. Alabama has no age-based standard deduction: the single standard deduction is $3,000 at this AGI. With the $1,500 exemption, total deductions are $4,500, leaving $234 taxable and $4.68 of tax."
-us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It claimed Alabama excludes federally tax-exempt interest. Alabama includes interest on other states' obligations, so the $4,734 is Alabama AGI. The $3,000 standard deduction and $1,500 exemption leave $234 taxable, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It included the $4,734 but claimed the standard deduction plus exemption is at least $5,000 for a single filer over 65. Alabama gives no age add-on, so the total is $3,000 plus $1,500, or $4,500. That leaves $234 taxed at 2%, for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It cited only the Social Security exemption and assumed income was below the filing threshold. The $4,734 of tax-exempt interest is Alabama AGI, which exceeds the $4,500 of combined deductions by $234. That $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It concluded there was no taxable income from the Social Security exemption alone. It ignored that Alabama taxes the $4,734 of tax-exempt interest. After $4,500 of deductions, $234 is taxable at 2%, for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It treated tax-exempt interest as excluded in Alabama. Alabama includes interest on non-Alabama obligations, so AGI is $4,734 and taxable income is $234 after the $3,000 standard deduction and $1,500 exemption. At 2%, the tax is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It asserted that taxable income was below the threshold without counting the $4,734 of tax-exempt interest, which Alabama taxes. That interest exceeds the $4,500 of combined deductions by $234, which is taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"It looked only at the Social Security exemption and set taxable income to zero. It ignored the $4,734 of tax-exempt interest, which Alabama includes in AGI. After $4,500 of deductions, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"It said the head has no other taxable income, but Alabama taxes the $4,734 of federally tax-exempt interest. After the $3,000 standard deduction and $1,500 exemption, $234 is taxable at 2%, for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"It treated the Social Security exemption as eliminating all taxable income and ignored the $4,734 of tax-exempt interest included in Alabama AGI. That AGI minus $4,500 of deductions leaves $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"It stated that tax-exempt interest is not subject to Alabama tax. Alabama includes interest on non-Alabama obligations in gross income, so AGI is $4,734 and taxable income is $234 after $4,500 of deductions. At 2%, that is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It claimed Alabama exempts federally tax-exempt interest and set AGI to $0. Alabama does not follow that exclusion, so AGI is $4,734. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"It left the tax-exempt interest out of the Alabama AGI base. Alabama includes that $4,734. Subtracting the $3,000 standard deduction and $1,500 exemption leaves $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It treated tax-exempt interest as non-Alabama-taxable. Alabama taxes interest on other states' obligations, so AGI is $4,734 and taxable income is $234 after $4,500 of deductions. At 2%, the tax is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It treated taxable income as effectively zero and never put the $4,734 of tax-exempt interest into Alabama AGI. Alabama taxes that interest. After the $3,000 standard deduction and $1,500 exemption, $234 remains, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It found no taxable Alabama income after the Social Security exemption. It missed that Alabama includes the $4,734 of tax-exempt interest in AGI, which exceeds the $4,500 of deductions and exemption by $234. At 2%, that is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It assumed tax-exempt interest produces no Alabama taxable income. Alabama adds the $4,734 into AGI, leaving $234 taxable after the $3,000 standard deduction and $1,500 exemption. At 2%, the tax is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It said the tax-exempt interest does not produce Alabama taxable income. Alabama taxes interest on non-Alabama obligations, so $4,734 of AGI minus $4,500 of deductions leaves $234. At 2%, that is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"It found no state-taxable income because it ignored that Alabama includes the $4,734 of federally tax-exempt interest in AGI. After $4,500 of deductions, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"It reported no other taxable income besides the exempt Social Security. The $4,734 of tax-exempt interest is Alabama-taxable. That leaves $234 after the $3,000 standard deduction and $1,500 exemption, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"It treated the tax-exempt interest as not producing Alabama taxable income. Alabama does not follow the federal muni exclusion, so AGI is $4,734 and taxable income is $234. At 2%, the tax is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"It said the tax-exempt interest produces no taxable income. Alabama includes that $4,734 in AGI, which exceeds the $4,500 of combined standard deduction and exemption by $234. At 2%, that is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,"It found no taxable state income because it never included the $4,734 of tax-exempt interest, which Alabama taxes. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It claimed there is no taxable income other than the exempt Social Security. It ignored the $4,734 of tax-exempt interest included in Alabama AGI, which leaves $234 taxable after $4,500 of deductions. At 2%, that is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It considered adding back the interest but assumed the deductions for a single filer age 65 or older fully offset $4,734. Alabama gives no age-based increase: the $3,000 standard deduction and $1,500 exemption total $4,500. That leaves $234 taxed at 2%, for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"It started from federal AGI of $0 and never added back the tax-exempt interest. Alabama includes that $4,734. The $3,000 standard deduction and $1,500 exemption then leave $234, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It treated the unspecified tax-exempt interest as Alabama-source and therefore exempt. Alabama AGI includes the $4,734, so taxable income is $234 after $4,500 of deductions. At 2%, that is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It assumed exemptions and deductions would be enough to cover the $4,734 of interest if included. Alabama allows only a $3,000 standard deduction and a $1,500 exemption, $4,500 in total. That leaves $234 taxable and $4.68 of tax."
-us,scenario_115,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It added back the tax-exempt interest but wrongly concluded that the standard deduction and personal exemption reduce taxable income to $0. They total only $4,500 ($3,000 plus $1,500), leaving $234 taxed at 2%, for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"It stated that tax-exempt interest is not taxable under Alabama law. Alabama includes interest on non-Alabama obligations, so AGI is $4,734 and taxable income is $234 after $4,500 of deductions. At 2%, that is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"It added back the interest but claimed the standard deduction plus $1,500 exemption is at least $5,500. Alabama's single standard deduction at this AGI is $3,000, so the total is $4,500. That leaves $234 taxed at 2%, for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It claimed Alabama has no state income tax. Alabama levies a graduated 2%, 4%, and 5% income tax. Here, $4,734 of Alabama AGI less $4,500 of deductions gives $234 taxed at 2%, or $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"It treated tax-exempt interest as nontaxable for Alabama and used federal AGI of $0. Alabama adds back the $4,734, leaving $234 taxable after $4,500 of deductions. At 2%, that is $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"It found no Alabama-taxable income besides Social Security. It missed that Alabama includes the $4,734 of federally tax-exempt interest in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It stated that Alabama exempts federally tax-exempt interest. Alabama taxes interest on other states' obligations, so AGI is $4,734 rather than $0. Taxable income is $234 after $4,500 of deductions, giving $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It labeled the $4,734 of tax-exempt interest not taxable in Alabama. Alabama includes it in AGI. After the $3,000 standard deduction and $1,500 exemption, $234 remains, taxed at 2% for $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It asserted that taxable income after the standard deduction is $0. With the $4,734 of interest in Alabama AGI, the $3,000 standard deduction and $1,500 exemption total only $4,500. That leaves $234 taxed at 2%, for $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"Assumed the $4,734 of federally tax-exempt interest is also excluded from Alabama income. Alabama taxes interest from other states' obligations, so Alabama AGI is $4,734. After the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, which is $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"Said tax-exempt interest is not taxable in Alabama. Alabama adds federally tax-exempt (non-Alabama) interest back into its AGI, so $4,734 less $4,500 of deductions leaves $234 taxed at 2%, or $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,state_local_rule,False,"Correctly excluded Social Security but concluded there was no other taxable income. It missed that Alabama includes the $4,734 of federally tax-exempt interest in Alabama AGI, which leaves $234 taxable after the $3,000 standard deduction and $1,500 exemption."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"Treated the tax-exempt interest as excluded and relied on nonexistent 'elderly exemptions' for a 77-year-old. Alabama taxes the $4,734 of interest and allows only a $3,000 standard deduction plus a $1,500 personal exemption, with no age add-on, so $234 is taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"Excluded the tax-exempt interest and assumed age-based deductions would cover income. Alabama includes the $4,734 of interest, and its $3,000 standard deduction plus $1,500 exemption total only $4,500, so $234 is taxed at 2%, or $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"Applied the federal exclusion for tax-exempt interest to Alabama. Alabama taxes interest from other states' obligations, so its AGI is $4,734, leaving $234 taxable after $4,500 of deductions and $4.68 of tax at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"Set Alabama AGI to $0 by excluding the $4,734 of tax-exempt interest. Alabama includes federally exempt municipal interest in its AGI, so after the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"Said Alabama does not tax tax-exempt interest. Alabama adds the $4,734 of federally tax-exempt interest to its AGI, which leaves $234 of taxable income and $4.68 of tax at the 2% rate."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,state_local_rule,False,"Excluded Social Security and then ignored the $4,734 of tax-exempt interest. Alabama counts that interest in its AGI, so $4,734 less $4,500 of standard deduction and exemption leaves $234 taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"Asserted that tax-exempt interest is not in Alabama gross income. Alabama taxes the $4,734 of federally exempt interest, so taxable income is $234 after the $3,000 standard deduction and $1,500 exemption, and the tax is $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"Treated the tax-exempt interest as not subject to Alabama tax. Alabama includes the $4,734 in its AGI, which leaves $234 taxable at 2% after $4,500 of deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"Carried federal AGI of $0 straight into Alabama. It missed Alabama's add-back of federally tax-exempt interest, which makes Alabama AGI $4,734 and leaves $234 taxable at 2% after the standard deduction and exemption."
+us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"Excluded the tax-exempt interest and invoked 'elderly deductions' that Alabama does not provide. Alabama AGI is $4,734 and the only offsets are the $3,000 standard deduction and $1,500 exemption, which leaves $234 taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"Said Alabama does not tax the tax-exempt interest. Alabama includes the $4,734 of federally exempt interest in its AGI, so $234 remains taxable after $4,500 of deductions, giving $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Correctly considered adding the interest to Alabama AGI but used a $4,500 'age 65+' standard deduction. Alabama's single standard deduction is $3,000 with no age add-on, so deductions total $4,500, not $6,000, and $234 remains taxable at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Excluded federal tax-exempt interest from Alabama income and assumed deductions zeroed out the rest. Alabama taxes the $4,734 of interest, and the $3,000 standard deduction plus $1,500 exemption leave $234 taxable at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"Recognized the $4,734 as Alabama income but claimed single filers over 65 get at least $5,000 in standard deduction plus exemption. Alabama allows $3,000 plus $1,500 = $4,500 with no age increase, which leaves $234 taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Asserted that taxable income is below the filing threshold. Alabama AGI includes the $4,734 of tax-exempt interest, which exceeds the $4,500 of standard deduction plus exemption by $234, taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Excluded Social Security and ignored the $4,734 of tax-exempt interest. Alabama taxes that interest, so taxable income is $234 after the $3,000 standard deduction and $1,500 exemption, and the tax is $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Said tax-exempt interest is excluded in Alabama. Alabama adds federally exempt interest back into its AGI, so $4,734 less $4,500 of deductions leaves $234 taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Concluded income was below the Alabama threshold without counting the $4,734 of tax-exempt interest. Alabama includes that interest in its AGI, and it exceeds the $4,500 of standard deduction plus exemption by $234, taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"Exempted Social Security and treated that as zeroing state income. It never added the $4,734 of federally tax-exempt interest to Alabama AGI, which leaves $234 taxable after $4,500 of deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"Claimed there was no other taxable income. Alabama taxes the $4,734 of federally tax-exempt interest, so after the $3,000 standard deduction and $1,500 exemption, $234 is taxed at 2%, or $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"Excluded Social Security and stopped there. It missed that Alabama AGI includes the $4,734 of tax-exempt interest, which yields $234 of taxable income after $4,500 of deductions and $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"Treated tax-exempt interest as not subject to Alabama tax. Alabama taxes federally exempt interest from other states' obligations, so $4,734 less $4,500 leaves $234 taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Said Alabama exempts federally tax-exempt interest and set Alabama AGI to $0. Alabama includes the $4,734 in its AGI, which leaves $234 taxable after the $3,000 standard deduction and $1,500 exemption."
+us,scenario_115,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"Left the tax-exempt interest out of Alabama AGI. Alabama adds it back, which makes AGI $4,734, and the $4,500 of standard deduction plus exemption leaves $234 taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"Treated the tax-exempt interest as non-Alabama-taxable. Alabama counts the $4,734 of federally exempt interest in its AGI, which leaves $234 taxable after $4,500 of deductions and $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Asserted that deductions reduce taxable income to zero. Alabama AGI is $4,734 of interest and the deductions are only $3,000 standard plus $1,500 exemption, so $234 remains taxable at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"Listed no taxable Alabama income after exempting Social Security. It missed that the $4,734 of federally tax-exempt interest is Alabama-taxable and exceeds the $4,500 of standard deduction plus exemption by $234."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Derived Alabama income from zero federal taxable income. Alabama adds the $4,734 of federally tax-exempt interest to its AGI, which leaves $234 taxable after the standard deduction and exemption and $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"Said the tax-exempt interest produces no Alabama taxable income. Alabama includes the $4,734 in its AGI, and only $4,500 of standard deduction plus exemption offsets it, which leaves $234 taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"Found no state-taxable income after excluding Social Security. It ignored that Alabama taxes the $4,734 of federally exempt interest, which yields $234 of taxable income and $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"Reported no other taxable income besides the excluded Social Security. Alabama AGI includes the $4,734 of tax-exempt interest, which leaves $234 taxable at 2% after $4,500 of deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"Said the tax-exempt interest yields no Alabama taxable income. Alabama adds it back in full, so $4,734 less the $3,000 standard deduction and $1,500 exemption leaves $234 taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"Treated the $4,734 of tax-exempt interest as non-taxable in Alabama. Alabama includes it in its AGI, so taxable income is $234 after $4,500 of deductions and the tax is $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,state_local_rule,False,"Identified no taxable state income after excluding Social Security. It missed that Alabama taxes the $4,734 of federally exempt interest, which leaves $234 taxable at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"Concluded there was no other taxable income besides Social Security. Alabama counts the $4,734 of tax-exempt interest in its AGI, which exceeds the $4,500 of standard deduction plus exemption by $234."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"Added the interest back but assumed the standard deduction and exemption for a single 65+ filer fully offset $4,734. Alabama gives $3,000 plus $1,500 = $4,500 with no age add-on, so $234 is taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"Started Alabama income from federal AGI of $0 and ignored the tax-exempt interest. Alabama adds that $4,734 back, and the $4,500 of deductions leave $234 taxable at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"Kept Alabama AGI at $0 because no out-of-state municipal-bond amount was identified. The listed $4,734 of tax-exempt interest is Alabama-taxable in full, which yields $234 of taxable income after the $3,000 standard deduction and $1,500 exemption."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Assumed exemptions and deductions would cover the interest if it were included. Alabama's $3,000 standard deduction plus $1,500 exemption total $4,500, which is $234 short of the $4,734 of interest, so $4.68 of tax is due at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"Added back the tax-exempt interest but claimed the standard deduction and personal exemption reduce it to zero. Alabama's $3,000 plus $1,500 = $4,500 leaves $234 of the $4,734 taxable at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"Said the $4,734 of tax-exempt interest is not taxable under Alabama law. Alabama includes federally exempt interest in its AGI, so $234 is taxable after $4,500 of deductions, giving $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"Added the interest back but claimed the standard deduction plus exemption is at least $5,500. Alabama's maximum single standard deduction is $3,000, so deductions total $4,500 and $234 remains taxable at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"Asserted that Alabama has no state income tax. Alabama levies a graduated income tax starting at 2%, and the $4,734 of tax-exempt interest less $4,500 of deductions leaves $234 taxable, or $4.68 of tax."
+us,scenario_115,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"Treated the tax-exempt interest as nontaxable and used AGI of 0. Alabama includes the $4,734 in its AGI, which leaves $234 taxable after the $3,000 standard deduction and $1,500 exemption."
+us,scenario_115,state_income_tax_before_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Found no Alabama-taxable income beyond the exempt Social Security. Alabama taxes the $4,734 of federally exempt interest, which leaves $234 taxable at 2% after $4,500 of deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"Said Alabama exempts federally tax-exempt interest. Alabama adds that interest to its AGI, so $4,734 less $4,500 of standard deduction plus exemption leaves $234 taxed at 2%, or $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"Labeled the $4,734 of tax-exempt interest as not taxable in Alabama. Alabama includes it in its AGI, which leaves $234 of taxable income after the $3,000 standard deduction and $1,500 exemption."
+us,scenario_115,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"Claimed the standard deduction reduces Alabama taxable income to $0. With the $4,734 of tax-exempt interest in Alabama AGI, the $3,000 standard deduction plus $1,500 exemption leaves $234 taxed at 2%."
us,scenario_115,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_116,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model improperly awarded a SNAP minimum allotment after its own benefit calculation reduced the regular allotment below zero, even though the household fails the gross-income eligibility test. It also submitted $6,804 despite stating an annual result of $3,402, doubling its own incorrectly calculated amount."
us,scenario_116,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model itself said the $52,800 in veterans benefits exceeds Florida's disabled-adult income limit, then answered eligible anyway because Florida 'typically provides expanded Medicaid' and has unnamed 'special provisions'. Florida is a non-expansion state with no such override. The spouse gets no SSI and fails the aged/disabled income test, so no category applies."
@@ -9871,112 +9978,114 @@ us,scenario_117,child3_early_head_start_eligible,gpt-5.4-mini,llm_error,categori
us,scenario_117,child3_early_head_start_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated remaining under age 3 throughout the year as sufficient to qualify as a participant. It omitted the required low-income condition, and this household's approximately $250,733 income is above the qualifying threshold."
us,scenario_117,child3_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 2 as sufficient for Head Start eligibility and omitted the income-eligibility test. The household's approximately $250,733 income exceeds the poverty-based Head Start limit, and no other qualifying pathway applies."
us,scenario_117,child3_wic_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated age under five as sufficient for WIC eligibility and omitted the separate income test. Although Child 3 meets the categorical age rule, household income exceeds the 185% federal-poverty-guideline threshold for a five-person household, yielding no WIC eligibility."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It phased out the overtime deduction from $150,000 MAGI, which is the single-filer threshold; the joint threshold is $300,000, so the full $25,000 applies, not $15,185. The adjudication permits its exclusion of the $2,589 refund because the prompt omits prior-year tax-benefit facts. It also skipped the $2,000 non-itemizer charitable deduction, and took a $1,200 CDCC even though Head has no earned income. On top of that, the $25,947 it submitted does not match its own $25,791 arithmetic."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It cut the overtime deduction to $14,927 using a $150,000 phase-out start; for joint filers the phase-out starts at $300,000 MAGI, so the full $25,000 applies. It also itemized $33,088 even though the standard deduction plus the $2,000 non-itemizer charitable deduction totals $34,200. Its taxable income of $202,719 should be $191,533.36."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It added the $35,525 overtime premium on top of gross wages that already include it, and it subtracted charity above the line. It used a $29,200 standard deduction and left out the $25,000 overtime deduction. It then submitted its pre-credit tax without subtracting the $6,600 nonrefundable CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It said tax was well above zero but submitted a placeholder of 0 without doing any computation. The correct path is $31,561.34 of tax on $191,533.36 of taxable income, minus the $6,600 CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It made up about $35,070 of mortgage interest from the loan balance at an assumed 7% rate (no interest amount is given, so it is $0) and added a $10,000 SALT deduction. It left out the $25,000 qualified overtime deduction and used 2025 brackets. It also treated the 16-year-old as a $500 other-dependent credit, though children under 17 get the $2,200 CTC, and it claimed a CDCC even though Head has no earnings."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Its reasoning reaches about $25,972, but it submitted $47,714, a number its own work does not support. That $25,972 also rests on made-up mortgage interest, no overtime deduction, a $2,000-per-child CTC, and a CDCC that is $0 because Head has no earnings."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It made up about $32,000 of mortgage interest and SALT to reach roughly $80,000 of itemized deductions. It also deducted the whole $35,525 overtime premium instead of the $25,000 joint cap. These overstated deductions pushed taxable income down to about $135,000 instead of $191,533.36."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It compared its $33,186 of itemized deductions with the $32,200 standard deduction alone. The frozen-reference calculation includes the 2026 $2,000 charitable deduction for non-itemizing joint filers, bringing the standard path to $34,200, which is about $1,014 above its itemized estimate. Its answer is $223.26 above the frozen reference for this excluded output."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 qualified overtime deduction and the $2,000 non-itemizer charitable deduction. It also used a $30,000 standard deduction and 2025 brackets instead of $32,200 and the 2026 brackets. It took a $2,000-per-child CTC and a $1,200 CDCC, although the CTC is $2,200 per child and the CDCC is $0 because Head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Its reasoning ends at $22,630, but it submitted an unrelated $62,700. The reasoning itself made up mortgage interest and SALT, and it left out the $25,000 overtime deduction and the $2,000 charitable deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It allowed only $12,500 of overtime deduction. The joint cap is $25,000, and with MAGI below the $300,000 joint phase-out start there is no reduction. It also itemized about $35,400 instead of taking the $32,200 standard deduction plus the $2,000 charitable deduction, which left taxable income at $202,834 instead of $191,533.36."
-us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It made up $32,565 of mortgage interest and left out the $25,000 overtime deduction. It allowed only $900 of CTC as nonrefundable, even though tax liability is large enough to absorb the full $6,600. It also took a $1,200 CDCC that is $0 because Head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It applied made-up 15%/25%/28% brackets and treated the CTC as fully phased out. OBBBA made the TCJA brackets and the $2,200 CTC (phase-out from $400,000) permanent. It also left out the $25,000 qualified overtime deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It removed the whole $35,525 overtime premium from AGI. The overtime deduction is a below-the-line deduction capped at $25,000, so AGI stays $250,733.36. It also overstated medical expenses and itemized instead of taking the $32,200 standard plus $2,000 charitable deduction, and it used $2,000 per child for the CTC instead of $2,200."
-us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only $600 of CTC as nonrefundable because it carved out a refundable portion first. With $31,561 of tax, the full $6,600 CTC is used as a nonrefundable credit. It also left out the $25,000 overtime deduction and $2,000 charitable deduction and used a $31,500 standard deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 rules: $5,000 personal exemptions, pre-TCJA brackets, and a $1,000-per-child CTC. OBBBA made the TCJA framework permanent, with the $2,200 CTC and no exemptions. It also left out the $25,000 overtime deduction and claimed a CDCC that is $0 because Head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $46,522 is higher than the pre-credit tax even on AGI less only the standard deduction, about $37,644. So it applied neither the $25,000 overtime deduction and $2,000 charitable deduction nor the $6,600 nonrefundable CTC. The correct figure is $31,561.34 − $6,600."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It computed tax under pre-TCJA rules that would apply had TCJA expired. OBBBA made the TCJA brackets, the $32,200 standard deduction, and the $2,200 CTC permanent. It left out the $25,000 overtime deduction and claimed a CDCC that is $0 because Head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA expired, using personal exemptions and a $1,000 CTC phased out to zero. Under 2026 law the $2,200-per-child CTC starts phasing out at $400,000 for joint filers, so all $6,600 is allowed. It also left out the $25,000 qualified overtime deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its $35,283 is consistent with taxing AGI less only a standard deduction, without the $25,000 qualified overtime deduction or $2,000 non-itemizer charitable deduction, and without the full $6,600 CTC. Correctly, those deductions bring taxable income to $191,533.36 and tax to $31,561.34 before the CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It cited a CTC phase-out that does not apply: the joint threshold is $400,000, above AGI under either refund reading. The frozen-reference calculation applies the full $6,600 against $31,561.34 of tax. Its $26,521.84 exceeds the frozen reference by $1,560.50; this output is excluded from scoring."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $41,320 is higher than tax with no overtime deduction even before credits. So it left out the $25,000 qualified overtime deduction and $2,000 charitable deduction and did not net the $6,600 CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used AGI around $248,145, consistent with excluding the $2,589 refund under a reading permitted by the adjudication. Its explanation does not specify its deduction or credit amounts. The frozen-reference calculation for this excluded output uses $191,533.36 of taxable income after the $32,200 standard deduction, $25,000 qualified overtime deduction and $2,000 non-itemizer charitable deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It deducted only an approximately $31,000 standard deduction and never applied the $25,000 qualified overtime deduction or the $2,000 non-itemizer charitable deduction, leaving taxable income at $219,734 instead of $191,533.36. It also used a $2,000-per-child CTC instead of $2,200."
-us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 qualified overtime deduction, which kept taxable income near $215,834. It also limited the nonrefundable CTC to $500 per child, although the full $2,200 per child is nonrefundable up to tax liability. And it took a $1,200 CDCC that is $0 because Head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It applied only a standard deduction, without the $25,000 qualified overtime deduction or $2,000 non-itemizer charitable deduction. It also claimed a childcare credit that is $0 because Head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed credits and deductions wipe out the tax. In fact $31,561.34 of tax on $191,533.36 of taxable income is offset only by the $6,600 CTC, leaving $24,961.34."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 qualified overtime deduction and reached taxable income of $214,996 instead of $191,533.36. It also itemized about $35,738 instead of taking the $32,200 standard deduction plus the $2,000 charitable deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Its figure works back to a $12,500 overtime deduction instead of the full $25,000 joint cap, with no $2,000 charitable deduction. It allowed CTC for only two children, though the 16-year-old also qualifies as under 17. It also took a $1,200 CDCC that is $0 because Head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its deductions and tax match the correct $31,561 path, but it subtracted a $600 CDCC. The CDCC earned-income limit uses the lower-earning spouse's earnings, and Head has $0, so the credit is $0."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized charity and SALT, about $33,900, instead of taking the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction ($34,200). That left taxable income about $305 too high."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $33,080 of charity (after the 0.5% floor) and state income tax, instead of taking the larger $34,200 from the standard deduction plus the $2,000 non-itemizer charitable deduction. That understated deductions by about $1,120."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized charity and state tax with an overstated SALT figure, reaching about $35,400. The correct path is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, which gives taxable income of $191,533.36."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It stopped at $193,534 of taxable income after the standard and overtime deductions. The omitted $2,000 charitable deduction adds $440 of tax at 22%; its answer exceeds the frozen reference by $440.14, including the effect of its rounded inputs. This output is excluded from scoring."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized charity and state tax with a SALT estimate that pushed the total just above $34,200. The correct path is the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, giving taxable income of $191,533.36."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $26,975 implies total deductions of about $50,050 instead of $59,200 ($32,200 standard + full $25,000 overtime + $2,000 charitable). That is consistent with an overtime deduction cut below the $25,000 joint cap, even though MAGI is under the $300,000 joint phase-out start."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Its $52,000 is higher than the pre-credit tax even with no overtime deduction. So it applied neither the $25,000 overtime deduction and $2,000 charitable deduction nor the $6,600 nonrefundable CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied TCJA-sunset rules: personal exemptions, a 25% bracket, and a CTC fully phased out. OBBBA made the TCJA brackets and the $2,200 CTC (phase-out from $400,000) permanent. It also left out the $25,000 overtime deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed TCJA sunset: $5,300 exemptions, 15%/25% brackets, uncapped SALT, and CTC phased out above $110,000. Under 2026 law there is a $32,200 standard deduction, a $25,000 overtime deduction, brackets topping out at 22% here, and the full $6,600 CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It got taxable income of about $191,534 right but miscalculated the bracket tax as $31,735. The 2026 joint brackets (10% to $24,800, 12% to $100,800, 22% above) give $31,561.34, so its tax is $174 too high."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It used reverted pre-TCJA brackets and personal exemptions, subtracted the ESI premium from wages, and treated the CTC as phased out. Under 2026 law the $6,600 CTC applies in full, the $25,000 overtime deduction applies, and the CDCC is $0 because Head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It took the overtime deduction above the line and measured the 7.5% medical floor against the reduced $225,734 AGI, which inflated medical to $3,116. It also deducted $237 of miscellaneous employee expenses that are disallowed. Its resulting $37,873 of itemized deductions replaced the correct $32,200 standard plus $2,000 charitable deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no answer and no explanation for this output. The correct path yields $31,561.34 of tax minus the $6,600 CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It compared its $34,414.79 of itemized deductions only with the $32,200 standard deduction, ignoring the $2,000 non-itemizer charitable deduction that makes the standard path $34,200. Its $9,027.51 Arkansas tax estimate is below the engine's modeled withholding proxy used for SALT. Including the $2,000 in its comparison would leave its $34,414.79 itemized estimate above $34,200. It reached $191,319.21 of taxable income; the frozen-reference calculation for this excluded output uses $191,533.36."
-us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed credits offset all federal income tax. In fact pre-credit tax is $31,561.34 and the only nonrefundable credit is the $6,600 CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"It allowed only $12,500 of overtime deduction instead of the full $25,000 joint cap, since MAGI is below $300,000. It itemized instead of taking the standard deduction plus $2,000 charitable. It also took a $1,200 CDCC that is $0 because Head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the 2026 $2,000 charitable deduction for non-itemizing joint filers, which added $440 of tax. It also subtracted a $1,200 CDCC even though Head's $0 earnings cap the credit at $0. The two steps reduce tax by a net $760; its answer is $759.86 below the frozen reference for this excluded output."
-us,scenario_117,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It reached roughly $31,580 of tax minus $6,600, then rounded the answer to $25,000 instead of computing it exactly. It also itemized $34,190, which is below the $34,200 from the standard deduction plus the non-itemizer charitable deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It made up $24,375 of mortgage interest at an assumed 6.5% rate on a $375,000 cap (the cap is $750,000, and no interest is listed, so it is $0) and added $10,000 of SALT to itemize $60,275. It also left out the $25,000 overtime deduction, used a $2,000-per-child CTC, and took a $600 CDCC that is $0 because Head has no earnings."
-us,scenario_117,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the whole overtime premium from wages and made up an 'overtime deduction credit', a $409,000 mortgage cap, and a $5,400 SALT cap. The correct treatment is a $25,000 below-the-line overtime deduction plus the $32,200 standard and $2,000 charitable deductions, which gives taxable income of $191,533.36."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It phased out the overtime deduction above $150,000 of AGI, which cut it to $15,185. The joint phase-out starts at $300,000, so the full $25,000 applies. It also left out the $2,000 non-itemizer charitable deduction and claimed a $1,200 CDCC even though Head has zero earned income, which limits creditable care expenses to $0."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It reduced the overtime deduction by 10% of MAGI over $150,000, down to $14,927. Joint filers do not start phasing out until $300,000, so the full $25,000 applies. It also itemized $33,088 instead of taking the larger standard-deduction path: $32,200 plus the $2,000 non-itemizer charitable deduction, or $34,200."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $34,165 because that beats the $32,200 standard deduction alone. It left out the 2026 joint non-itemizer charitable deduction of $2,000, which makes the standard path $34,200. That $35 gap in deductions accounts for the $7.80 difference in tax."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It added the $35,525 overtime premium on top of gross wages that already include it, and it subtracted charitable gifts above the line. It used a stale $29,200 standard deduction and never applied the $25,000 overtime deduction. It also never subtracted the $6,600 CTC in any clear way."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its own reasoning says tax is well above zero, but it submitted 0 without doing any calculation. The correct figure is $31,561 of tax on $191,533 of taxable income, minus the $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It imputed $35,070 of mortgage interest from the $501,000 balance even though no interest was listed, so that input is 0. It also claimed $10,000 of SALT with no state taxes listed and never applied the $25,000 overtime deduction. On credits, it treated the 16-year-old as a $500 ODC dependent rather than a $2,200 CTC child and claimed a $600 CDCC despite Head having no earnings."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Its reasoning arrived at about $25,972, but it submitted $47,714, which matches nothing it calculated. That reasoning was also built on $20,000 of imputed mortgage interest, no overtime deduction, a $2,000-per-child CTC, and a CDCC that Head's zero earnings rule out."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It made up about $80,000 of itemized deductions from mortgage interest and SALT that were never listed. It then subtracted the entire $35,525 overtime premium instead of the $25,000 joint cap, which pushed taxable income down to about $135,000 instead of $191,533. It also used a $2,000-per-child CTC and an unavailable CDCC."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized $33,186 because that beats the $32,200 standard deduction. It ignored the $2,000 joint non-itemizer charitable deduction that comes with the standard deduction, so the standard path is worth $34,200. That error raised taxable income by $1,015."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It used a $30,000 standard deduction and 2025 brackets, and it never applied the $25,000 qualified-overtime deduction or the $2,000 charitable deduction, which left taxable income at $220,734. It also used a $2,000-per-child CTC instead of $2,200 and subtracted a $1,200 CDCC that Head's zero earned income rules out."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Its reasoning reached $22,630, but it submitted $62,700, a number it never derived. The reasoning itself imputed $32,000 of mortgage interest and $10,000 of SALT that were never listed, skipped the $25,000 overtime deduction, and claimed a $1,200 CDCC."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied a $12,500 overtime deduction, which is the single-filer cap. A joint return gets up to $25,000, and the $35,525 premium more than covers it. It also itemized $35,400 instead of the $34,200 standard path, so taxable income came out at $202,834 instead of $191,533."
+us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It imputed $32,565 of mortgage interest from the balance alone and left out the $25,000 overtime deduction. It subtracted a $1,200 CDCC that Head's zero earnings rule out. It also allowed only $900 of nonrefundable CTC, when the full $6,600 is absorbed against tax liability."
+us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA brackets (15%, 25%, 28%) and treated the CTC as fully phased out. The permanent TCJA/OBBBA rates still apply for 2026, as does the $2,200 CTC with a $400,000 joint phase-out. It also left out the $25,000 overtime deduction."
+us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the whole $35,525 overtime premium from AGI. The actual rule is a deduction from taxable income capped at $25,000 for joint filers. It also inflated medical deductions to $3,905 and used a $2,000-per-child CTC instead of $2,200, ending at $22,816 instead of $24,961."
+us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used a $31,500 standard deduction and left out both the $25,000 overtime deduction and the $2,000 charitable deduction. It then treated $5,400 of the CTC as refundable and allowed only $600 against tax. In fact the nonrefundable CTC absorbs the full $6,600, because tax liability far exceeds it."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired, so it used $5,000 personal exemptions, pre-TCJA rates, and a $1,000-per-child CTC. In 2026 the law still gives a $32,200 standard deduction, a $25,000 overtime deduction, and a $2,200 CTC. It also subtracted a $1,200 CDCC that Head's zero earnings rule out."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Its $46,522 is higher than the $37,644 of tax owed with only the standard deduction and no credits. That means it left out the $25,000 overtime deduction and the $2,000 charitable deduction, did not subtract the $6,600 CTC, and overstated the bracket tax."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It applied pre-TCJA 2026 rules: higher brackets, personal exemptions, and the old CTC. Current law instead gives a $32,200 standard deduction, a $25,000 overtime deduction, the $2,000 charitable deduction, and a $2,200 CTC per child. It also subtracted a CDCC that Head's lack of earnings rules out."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA had expired and said a $1,000 CTC was fully phased out. Under current law, the three children get $2,200 each with no phase-out below $400,000 of joint MAGI. It also omitted the $25,000 overtime deduction."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"Its $35,283 is close to the tax on about $218,000 of taxable income, so it skipped the $25,000 overtime deduction and the $2,000 charitable deduction. It also did not fully subtract the $6,600 CTC from the $31,561 of tax on $191,533."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied a CTC phase-out 'at higher income levels.' The joint phase-out starts at $400,000 of MAGI, so all $6,600 of CTC is allowed. Its $26,522 also implies about $7,000 too little in deductions compared with the $59,200 total (standard deduction, overtime, and charitable)."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Its $41,320 is higher than the tax owed with the standard deduction alone and no credits. That means it left out the $25,000 overtime deduction and the $2,000 charitable deduction and did not subtract the $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It left the taxable $2,589 state refund out of AGI ($248,145 instead of $250,734). Its $33,816 shows it did not apply the $25,000 overtime deduction and did not subtract the full $6,600 CTC from the tax."
+us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It applied only a standard deduction of about $31,000 and left out the $25,000 qualified-overtime deduction and the $2,000 charitable deduction, so taxable income came out at $219,734 instead of $191,533. It also used a $2,000-per-child CTC instead of $2,200."
+us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $25,000 overtime deduction, which left taxable income at about $215,834. It limited the nonrefundable CTC to $500 per child, when the full $2,200 per child offsets tax liability. It also subtracted a $1,200 CDCC that Head's zero earnings rule out."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It applied only the standard deduction and never took the $25,000 qualified-overtime deduction or the $2,000 non-itemizer charitable deduction. It also counted a childcare credit that Head's zero earned income rules out."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed that deductions and credits wipe out the tax, but $191,533 of taxable income produces $31,561 of tax. The only nonrefundable credit, the $6,600 CTC, cannot bring that to zero."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It never applied the $25,000 qualified-overtime deduction, which left taxable income at $214,996 instead of $191,533. That deduction is allowed in addition to the standard deduction or itemized deductions, and the $35,525 FLSA premium exceeds the cap."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It used the $12,500 single-filer overtime cap instead of the $25,000 joint cap and left out the $2,000 charitable deduction, which gave taxable income of $206,034. It counted only two children for the CTC, although the 16-year-old also qualifies. It also subtracted a $1,200 CDCC that Head's zero earnings rule out."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income and $6,600 CTC match the correct calculation. However, it subtracted a $600 CDCC. Creditable care expenses cannot exceed the lower-earning spouse's earned income, and Head's is $0, so the CDCC is $0."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $33,900 of charitable and SALT deductions. The standard-deduction path is worth more: $32,200 plus the $2,000 joint non-itemizer charitable deduction, or $34,200. Choosing to itemize raised taxable income by about $300."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $33,080, including the charitable deduction after the 0.5%-of-AGI floor and the state income tax. That is less than the $34,200 available from the $32,200 standard deduction plus the $2,000 joint non-itemizer charitable deduction, which it ignored."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $35,400, overstating the Arkansas income-tax SALT component. The household's itemizable deductions fall below $34,200, the $32,200 standard deduction plus the $2,000 joint non-itemizer charitable deduction, so the standard path applies."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It correctly took the $32,200 standard deduction and the $25,000 overtime deduction, but it left out the 2026 non-itemizer charitable deduction ($2,000 for joint filers). That made taxable income $193,534 instead of $191,534, which is $440 of extra tax."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It itemized about $34,340 by overstating the state-tax SALT deduction. The household's itemizable deductions fall below $34,200, the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, so it should have taken the standard path."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Its $26,975 implies about $50,000 of total deductions instead of $59,200. That roughly $9,000 shortfall comes from cutting the overtime deduction to about $15,000, phasing it out from $150,000 instead of the $300,000 joint MAGI threshold."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Its $52,000 is higher than the tax owed even with only the standard deduction and no credits. That means it left out the $25,000 overtime deduction, the $2,000 charitable deduction, and the $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied post-TCJA-sunset rules: personal exemptions, the 25% bracket, and a fully phased-out CTC. Current 2026 law keeps the TCJA brackets, the $2,200 CTC with a $400,000 joint phase-out, and the $25,000 overtime deduction."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed the TCJA sunset, which brought back $5,300 exemptions, 15%/25% brackets, and a CTC phase-out at $110,000. It also allowed uncapped SALT. Current 2026 law uses a $32,200 standard deduction, a $25,000 overtime deduction, the TCJA brackets, and a $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Its taxable income of $191,534 is right, but its bracket tax of $31,735 is wrong. The correct figure is $2,480 + $9,120 + 22% × $90,734 = $31,561.48, so the result overstates tax by about $174."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It used pre-TCJA brackets and personal exemptions, said the CTC was phased out, and subtracted the employer-sponsored premium from wages. It also claimed a $1,200 CDCC. Current law keeps the TCJA structure with a $6,600 CTC and a $25,000 overtime deduction, and Head's zero earnings rule out the CDCC."
+us,scenario_117,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It treated the overtime deduction as reducing AGI, to $225,734. Using that lower AGI shrank the 7.5% medical floor and inflated medical expenses to $3,116, and it also added $237 of suspended miscellaneous deductions. The resulting $37,873 of itemized deductions replaced the correct $34,200 standard path."
+us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no answer and no reasoning for this output, so there is no value to score."
+us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"It counted Head's $8,223 premium twice: the 'excluding Medicare Part B' amount and the 'other premiums' amount are the same premium. That created a $740.95 medical deduction that does not exist. Without it, itemized deductions are $33,674, less than the $34,200 standard path, so the standard deduction applies."
+us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed credits offset the entire tax. The only nonrefundable credit is the $6,600 CTC, which cannot cancel $31,561 of tax on $191,533 of taxable income."
+us,scenario_117,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"It applied a $12,500 overtime deduction, which is the single-filer cap, instead of the $25,000 joint cap. It also subtracted a $1,200 CDCC, even though the lower earner's (Head's) $0 earned income limits creditable expenses to zero."
+us,scenario_117,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It subtracted a $1,200 CDCC, even though Head has no earned income, which caps creditable expenses at $0. It also left out the $2,000 joint non-itemizer charitable deduction, which put taxable income at $193,534 instead of $191,534."
+us,scenario_117,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated itemized deductions of about $34,190 and tax of about $24,980. It then rounded the answer to $25,000 instead of reporting the calculated figure. The standard path of $32,200 plus the $2,000 charitable deduction ($34,200) gives exactly $24,961.34."
+us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It imputed $24,375 of mortgage interest from the balance alone, though no interest was listed so it is 0, and claimed $10,000 of SALT. It never applied the $25,000 overtime deduction. It also used a $2,000-per-child CTC and a $600 CDCC that Head's zero earnings rule out."
+us,scenario_117,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It excluded the overtime premium from wages and also made up rules that do not exist: an 'additional standard deduction for net overtime,' an 'overtime deduction credit,' and a $409,000 mortgage cap. None of these correspond to how the $25,000 below-the-line overtime deduction works."
us,scenario_117,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"It never applied the rule that the ACTC can refund only the CTC left after offsetting tax liability. A federal tax bill of more than $25,000 absorbs all of this household's CTC, so the ACTC is $0. Instead it invented a $3,750 amount 'limited by the ratio of Additional CTC to total CTC'. Along the way it counted the FLSA overtime premium twice on top of gross wages, used the 2023 $27,700 standard deduction and $2,000-per-child CTC, and said income was above the $400,000 phase-out when AGI is about $250,733."
us,scenario_117,federal_refundable_credits,deepseek-v4-flash-0731,llm_error,other,False,"It treated the $1,700-per-child refundable cap as an amount paid out automatically ($5,100). It skipped the step that limits the ACTC to CTC left after tax liability. With more than $25,000 of federal income tax, the full $6,600 CTC is used as a nonrefundable credit, so the refundable portion is $0."
us,scenario_117,federal_refundable_credits,deepseek-v4.1-flash,llm_error,other,False,"It multiplied 3 children by an $1,800 refundable cap to get $5,400. It never limited the ACTC to CTC left after the credit offsets tax liability. The household's federal tax of more than $25,000 absorbs the full $6,600 CTC, so the ACTC is $0. EITC is also $0."
us,scenario_117,federal_refundable_credits,glm-5.3,llm_error,other,False,"It stated the rule backwards, saying that 'with ample tax liability' the household claims the full refundable CTC of $1,700 per child ($5,100). In fact, ample tax liability is exactly what reduces the ACTC to zero: the refundable portion is limited to CTC left after offsetting tax. More than $25,000 of federal tax absorbs the entire $6,600 CTC, so refundable credits are $0."
us,scenario_117,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for federal_refundable_credits, so there is no substantive calculation to score. Under the correct derivation, the full $6,600 CTC offsets this household's more-than-$25,000 federal tax, which leaves a $0 ACTC. EITC is $0 as well."
-us,scenario_117,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"The model correctly found that income is above 130% FPL with no SNAP/TANF categorical eligibility, but it treated that as the only way to get free meals. It never applied Arkansas Act 123 of 2025, which gives the two K-12 children a free daily breakfast worth $741.60 a year. That makes free school meal support positive."
-us,scenario_117,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model concluded there was no free school meal support from the 130% FPL test and the lack of categorical eligibility alone. It missed Arkansas's universal free breakfast for public school students (A.C.A. § 6-18-722(f)), which puts breakfast in the free tier for the 16- and 11-year-old."
-us,scenario_117,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model added the FLSA overtime premium on top of gross wages, which double-counts it because the wages already include overtime. It also understated 130% FPL as about $34,450, when the correct figure is about $50,000. The decisive error is that it applied only the income test and ignored Arkansas Act 123's free daily breakfast for every public school student, which gives the two school-age children free breakfast regardless of income."
-us,scenario_117,free_school_meals_eligible,claude-haiku-5.5,llm_error,state_local_rule,False,"The model used only the income limit for free meals. It missed that Arkansas, starting in 2025-26, gives one free breakfast a day to every public school student regardless of income (Act 123 of 2025). That gives the two K-12 children $741.60 a year in free breakfast support."
-us,scenario_117,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model correctly placed the household above 130% FPL but missed Arkansas's statewide free breakfast under A.C.A. § 6-18-722(f). Under that rule, breakfast is free for the 16- and 11-year-old even though lunch stays paid, so free meal support is positive."
-us,scenario_117,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model counted the overtime premium on top of wages, which double-counts it, and assumed the federal 130% FPL test settles eligibility. It never applied Arkansas Act 123 of 2025, which gives each public school student a free breakfast at no cost and makes breakfast free for the household's two K-12 children."
-us,scenario_117,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model applied only the 130% FPL free-meal income test. It ignored Arkansas's income-independent free breakfast for all public school students starting in 2025-26, which gives the two school-age children $741.60 a year in free breakfast support."
-us,scenario_117,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"The model ruled the household ineligible because income is far above 130% FPL. It overlooked Arkansas Act 123 of 2025 (A.C.A. § 6-18-722(f)), under which every public school student gets one free breakfast a day, so the two K-12 children receive positive free meal support."
-us,scenario_117,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model treated NSLP income eligibility at 130% FPL as the only route to free meals. It missed Arkansas's state universal breakfast law, which puts breakfast in the free tier for the 16- and 11-year-old regardless of the $248,675 income."
-us,scenario_117,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"The model gave the free-meal threshold as 185% FPL, which is the reduced-price limit, not the free limit. More importantly, it did not apply Arkansas Act 123 of 2025, under which every public school student receives one free breakfast a day. That gives the two K-12 children $741.60 a year in free breakfast support."
-us,scenario_117,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model used only the federal income threshold. It missed Arkansas's free daily breakfast for all public school students under A.C.A. § 6-18-722(f), which makes breakfast free for the household's two school-age children."
-us,scenario_117,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model applied only the 130% FPL free-meal test. It ignored Arkansas Act 123 of 2025, which from 2025-26 gives each public school student a free breakfast regardless of income, so the two K-12 children have positive free meal support."
-us,scenario_117,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model stopped at the 130% poverty income test. It missed Arkansas's statewide free breakfast for public school students, which puts breakfast in the free tier for the 16- and 11-year-old and is worth $741.60 a year."
-us,scenario_117,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model correctly found no income-based or SNAP/TANF categorical route to free meals. It then left out the separate Arkansas Act 123 route: every public school student gets one free breakfast a day, so the two K-12 children receive free breakfast support."
-us,scenario_117,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model applied only the income limit. It did not account for Arkansas's universal free breakfast (A.C.A. § 6-18-722(f)), which gives the household's two school-age children free breakfast regardless of income."
-us,scenario_117,free_school_meals_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model counted the $35,525 overtime premium on top of wages that already include it; its reported $283,670 equals wages plus that premium less the $530 short-term capital loss. It then applied only the 130% FPL test. It missed Arkansas Act 123 of 2025, under which breakfast is free for every public school student, giving the two K-12 children $741.60 a year in free meal support."
-us,scenario_117,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model applied only the income thresholds. It missed Arkansas's income-independent free breakfast for all public school students starting in 2025-26, which makes breakfast free for the 16- and 11-year-old."
-us,scenario_117,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model treated the income limits as decisive. It ignored Arkansas Act 123 of 2025 (A.C.A. § 6-18-722(f)), which gives each public school student a free breakfast and puts breakfast in the free tier for the two K-12 children."
-us,scenario_117,free_school_meals_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,"The model applied only the 130% FPL threshold. It missed that Arkansas gives a free daily breakfast to every public school student regardless of income, which gives the household positive free meal support for its two school-age children."
-us,scenario_117,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model used only the income threshold. It did not apply Arkansas's universal school breakfast under Act 123 of 2025, which gives the 16- and 11-year-old free breakfast worth $741.60 a year."
-us,scenario_117,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"The model based its answer only on the income eligibility threshold. It overlooked Arkansas's statewide free breakfast for public school students starting in 2025-26, which makes breakfast free for the two K-12 children."
-us,scenario_117,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"The model applied only the 130% FPL free-meal test. It missed Arkansas Act 123 of 2025, which gives one free breakfast a day to every public school student, so the household's two school-age children have positive free meal support."
-us,scenario_117,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"The model stopped at the 130% FPL income test. It ignored A.C.A. § 6-18-722(f), under which Arkansas public school students receive free breakfast regardless of income, putting breakfast in the free tier for the 16- and 11-year-old."
-us,scenario_117,free_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,"The model applied only the 130% poverty guideline test. It missed Arkansas's universal free breakfast for public school students (Act 123 of 2025), which gives the two K-12 children $741.60 a year in free breakfast support."
-us,scenario_117,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model compared $248,675 to about $48,945 (130% FPL) and treated that as decisive. It never applied Arkansas Act 123 of 2025, which gives every public school student a free daily breakfast and makes breakfast free for the household's two school-age children."
-us,scenario_117,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model set free meal support to $0 using only the 130% poverty income test. It missed Arkansas's universal free breakfast, which provides $741.60 a year in free breakfast for the 16- and 11-year-old."
-us,scenario_117,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,"The model applied only the free-meal income threshold. It ignored Arkansas's income-independent free breakfast for all public school students starting in 2025-26, so the two K-12 children have positive free meal support."
-us,scenario_117,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"The model took the absence of income-based or categorical free-meal indicators to mean zero support. It missed Arkansas Act 123 of 2025, under which every public school student receives a free breakfast, so the household's two school-age children qualify."
-us,scenario_117,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model correctly ruled out the income and SNAP/TANF routes. It then missed the Arkansas state route (A.C.A. § 6-18-722(f)), which gives every public school student one free breakfast a day and puts breakfast in the free tier for the 16- and 11-year-old."
-us,scenario_117,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model applied only the income threshold. It did not account for Arkansas's statewide free breakfast for public school students, which gives the two K-12 children $741.60 a year in free breakfast support regardless of income."
-us,scenario_117,free_school_meals_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,"The model checked only the federal free-meal income threshold and categorical eligibility. It omitted Arkansas Act 123 of 2025, which provides a free daily breakfast to every public school student and makes free meal support positive for the two school-age children."
-us,scenario_117,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model used only the income threshold. It missed Arkansas's universal free breakfast starting in 2025-26, under which the household's 16- and 11-year-old receive breakfast in the free tier."
-us,scenario_117,free_school_meals_eligible,gpt-6-astra,llm_error,state_local_rule,False,"The model applied only the federal income-tested rules: above 130% FPL and no categorical benefit. It left out Arkansas's non-income-tested free breakfast under A.C.A. § 6-18-722(f), which gives the two K-12 children free breakfast worth $741.60 a year."
-us,scenario_117,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"The model applied only the federal free-meal income limit. It ignored Arkansas Act 123 of 2025, which gives every public school student one free breakfast a day, so the two school-age children receive free meal support."
-us,scenario_117,free_school_meals_eligible,gpt-6-sol,llm_error,state_local_rule,False,"The model ruled out support based on income alone. It missed that Arkansas gives a free daily breakfast to all public school students regardless of income, which puts breakfast in the free tier for the household's two K-12 children."
-us,scenario_117,free_school_meals_eligible,gpt-6.1-sol,llm_error,state_local_rule,False,"The model correctly ruled out income-based and categorical free meals. It never applied Arkansas Act 123 of 2025, which gives each public school student a free breakfast and yields $741.60 a year in free breakfast support for the 16- and 11-year-old."
-us,scenario_117,free_school_meals_eligible,gpt-6.1-terra,llm_error,state_local_rule,False,"The model applied the income threshold and categorical tests only. It missed Arkansas's statewide free breakfast for public school students (A.C.A. § 6-18-722(f)), which makes breakfast free for the two school-age children."
-us,scenario_117,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,"The model treated high income as ruling out free meals. Arkansas Act 123 of 2025 gives every public school student a free daily breakfast regardless of income, so the two K-12 children have positive free meal support."
-us,scenario_117,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model applied only the 130% FPL limit and returned zero free meal value. It missed Arkansas's universal free breakfast, which PolicyEngine values at $741.60 a year for the household's two school-age children."
-us,scenario_117,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"The model checked the 130% FPL test and SNAP/TANF/Medicaid categorical eligibility. It never applied Arkansas Act 123 of 2025, under which every public school student receives a free breakfast, so the 16- and 11-year-old get breakfast in the free tier."
-us,scenario_117,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"The model correctly found no income-based or categorical eligibility but missed the separate Arkansas route. A.C.A. § 6-18-722(f) gives every public school student a free breakfast starting in 2025-26, so the two K-12 children have positive free meal support."
-us,scenario_117,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model applied only the 130% FPL threshold. It ignored Arkansas's income-independent free breakfast for public school students, which gives the household's two school-age children $741.60 a year in free breakfast support."
-us,scenario_117,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,The model explicitly stated that Arkansas does not provide universal free school meals. That is wrong for breakfast: Act 123 of 2025 (A.C.A. § 6-18-722(f)) gives every public school student one free breakfast a day starting in 2025-26. That puts breakfast in the free tier for the 16- and 11-year-old even though lunch stays paid.
-us,scenario_117,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"The model correctly placed income above 130% FPL with no categorical eligibility. It missed Arkansas Act 123 of 2025, which gives each public school student a free breakfast regardless of income, so the two K-12 children receive free breakfast support."
-us,scenario_117,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model applied only the 130% FPL free-meal limit. It ignored Arkansas's statewide free breakfast for public school students, which makes breakfast free for the household's 16- and 11-year-old."
-us,scenario_117,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,"The model ruled the household out on income alone. It never applied Arkansas Act 123 of 2025, which gives every public school student one free breakfast a day and yields positive free meal support for the two school-age children."
-us,scenario_117,free_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,"The model checked the 130% FPL test and direct certification only. It missed Arkansas's universal free breakfast under A.C.A. § 6-18-722(f), which gives the two K-12 children free breakfast regardless of income."
-us,scenario_117,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"The model judged eligibility only against the free-meal income limit. It missed Arkansas Act 123 of 2025, under which every public school student receives a free breakfast, giving the household $741.60 a year in free meal support for its two school-age children."
-us,scenario_117,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model treated the 130% FPL test and SNAP/TANF participation as the only routes to free meals. It ignored Arkansas's statewide free breakfast for all public school students starting in 2025-26, which puts breakfast in the free tier for the 16- and 11-year-old."
-us,scenario_117,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"The model compared wages to the USDA free-meal income guideline and stopped there. It missed Arkansas Act 123 of 2025, which gives every public school student one free breakfast a day regardless of income, so the two K-12 children have positive free meal support."
-us,scenario_117,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,"The model applied only the income limits. It did not account for Arkansas's universal free breakfast (A.C.A. § 6-18-722(f)), which gives the household's two school-age children $741.60 a year in free breakfast support."
+us,scenario_117,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"Correctly found that income is far above 130% of poverty and that there is no SNAP/TANF categorical eligibility, but it treated the federal income tier as the only route to free meals. It missed Arkansas Act 123 of 2025, which gives each public school student one free breakfast a day regardless of income. For the 16- and 11-year-old, that breakfast is $741.60 of free meal support, so the answer is 1."
+us,scenario_117,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"It looked only at the 130% FPL test and categorical eligibility, which correctly put lunch in the paid tier. It never applied Arkansas's 2025-26 free breakfast for all public school students (A.C.A. § 6-18-722(f)). That breakfast gives the two school-age children $741.60 a year of free meal support."
+us,scenario_117,free_school_meals_eligible,claude-fable-5.2,llm_error,state_local_rule,False,"It answered from the 130%-of-poverty income test alone. It missed Arkansas Act 123 of 2025, which provides one free breakfast a day to every public school student regardless of income. Because of that breakfast, the household's two K-12 children receive $741.60 of free meal support, and the output is 1."
+us,scenario_117,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"It counted income wrongly: it added the $35,525 overtime premium on top of wages that already include it, and it understated 130% FPL as $34,450 when 130% of $38,680 is about $50,284. Its main error is ignoring Arkansas Act 123 of 2025's universal free breakfast for public school students. That breakfast makes free meal support positive ($741.60 for the two K-12 children) at any income."
+us,scenario_117,free_school_meals_eligible,claude-haiku-5.5,llm_error,state_local_rule,False,"It stopped once income exceeded the federal free-meal limit. It did not apply Arkansas's no-cost daily breakfast for every public school student from 2025-26 (A.C.A. § 6-18-722(f)). That breakfast puts the two school-age children's breakfast in the free tier, worth $741.60, so the answer is 1."
+us,scenario_117,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"It correctly compared the $248k income with the 130% FPL limit, but income-tested lunch is not the only free-meal route in Arkansas. Act 123 of 2025 gives each public school student one free breakfast a day regardless of income. For the 16- and 11-year-old, that is $741.60 of free support."
+us,scenario_117,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"It based eligibility only on household income against the 130% FPL guideline. It missed that Arkansas, from the 2025-26 school year, gives a free breakfast to every public school student who doesn't otherwise get one (Act 123 of 2025). That breakfast gives the two K-12 children $741.60 of free meal support, so the value is 1."
+us,scenario_117,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"It treated being above 130% FPL as settling the answer. It never considered Arkansas's universal free breakfast under A.C.A. § 6-18-722(f), which applies regardless of income. That breakfast makes free meal support positive ($2.06 a day × 180 days × 2 children = $741.60)."
+us,scenario_117,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"It applied only the federal 130% FPL free-meal test. It missed Arkansas Act 123 of 2025, which gives every public school student one free breakfast a day from 2025-26. That breakfast gives the household's two school-age children $741.60 of free meal support."
+us,scenario_117,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"It treated the National School Lunch Program's 130% FPL test as the only route to free meals. It ignored Arkansas's 2025-26 law giving every public school student a free daily breakfast regardless of income. Because of that law, breakfast for the 16- and 11-year-old is free, worth $741.60, so the answer is 1."
+us,scenario_117,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"It called 185% FPL the free-meal threshold; that is actually the reduced-price limit, and free is 130%. More importantly, it treated the income test and SNAP/TANF as the only routes. Arkansas Act 123 of 2025 makes one breakfast a day free for every public school student, giving the two K-12 children $741.60 of free support."
+us,scenario_117,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,It answered from the federal free-meal income threshold alone. It missed Arkansas's universal free breakfast for public school students under Act 123 of 2025. That breakfast makes free support positive at any income ($741.60 for the two school-age children).
+us,scenario_117,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It applied only the 130% FPL free-meal limit. It did not account for Arkansas's no-cost daily breakfast for every public school student from 2025-26 (A.C.A. § 6-18-722(f)). That breakfast puts the two K-12 children's breakfast in the free tier, worth $741.60, so the value is 1."
+us,scenario_117,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"It treated being above the 130% poverty limit as decisive. It ignored Arkansas Act 123 of 2025, which provides one free breakfast a day to all public school students regardless of income. That breakfast gives the 16- and 11-year-old $741.60 of free meal support."
+us,scenario_117,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It correctly ruled out the income-based and SNAP/TANF categorical routes, but it missed Arkansas's state-funded universal breakfast (Act 123 of 2025, A.C.A. § 6-18-722(f)). That breakfast is free for each public school student who doesn't already get one. The two K-12 children therefore receive $741.60 of free breakfast support, so the answer is 1."
+us,scenario_117,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It stopped at the federal income limit for free meals. It never applied Arkansas's 2025-26 free breakfast for every public school student, which ignores income. That breakfast gives the household's two school-age children $741.60 a year of free meal support."
+us,scenario_117,free_school_meals_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"It counted income as $283,670 by adding the overtime premium on top of wages that already include it, and it put 130% FPL at $47,554. The answer turns on a different rule: Arkansas Act 123 of 2025 gives every public school student a free daily breakfast. That breakfast yields $741.60 of free support for the two K-12 children at any income."
+us,scenario_117,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It applied only the income eligibility thresholds. It missed Arkansas's universal free breakfast for public school students from the 2025-26 school year (A.C.A. § 6-18-722(f)). That breakfast gives the two school-age children $741.60 of free meal support, so the value is 1."
+us,scenario_117,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It judged eligibility only against the income-tested free-meal limits. It missed Arkansas Act 123 of 2025, which gives each public school student one free breakfast a day regardless of income. That breakfast puts breakfast in the free tier for the 16- and 11-year-old, worth $741.60."
+us,scenario_117,free_school_meals_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,"It treated the 130% FPL test as the only route to free meals. It did not apply Arkansas's universal free breakfast under A.C.A. § 6-18-722(f), which covers every public school student from 2025-26. That breakfast gives the two K-12 children $741.60 of free support, so the answer is 1."
+us,scenario_117,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It claimed the income threshold rules out free support under PolicyEngine, but it missed that PolicyEngine models Arkansas Act 123 of 2025. That law gives every public school student one free breakfast a day regardless of income. For the two school-age children, that is $741.60 of free breakfast support."
+us,scenario_117,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"It applied only the income eligibility threshold. It ignored Arkansas's 2025-26 universal no-cost breakfast for public school students, which makes breakfast free for the 16- and 11-year-old ($741.60 a year). Free school meal support is therefore positive."
+us,scenario_117,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"It used only the 130% FPL free-meal test. It missed Arkansas Act 123 of 2025, which gives a free daily breakfast to every public school student who doesn't otherwise get one. That breakfast gives the two K-12 children $741.60 of free support, so the answer is 1."
+us,scenario_117,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"It stopped at the 130% FPL comparison. It did not apply Arkansas's universal free breakfast (A.C.A. § 6-18-722(f)), which ignores income. With two public school-age children, the household receives $741.60 of free breakfast support."
+us,scenario_117,free_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,It treated being above 130% of the poverty guideline as ruling out any free meal support. It missed Arkansas's 2025-26 free breakfast for every public school student (Act 123 of 2025). That breakfast gives the 16- and 11-year-old $741.60 of free support.
+us,scenario_117,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"It correctly found that the $248,675 in wages exceeds the 130% FPL limit for five people, but it treated that limit as the only route to free meals. Arkansas Act 123 of 2025 gives every public school student one free breakfast a day regardless of income. That breakfast gives the two K-12 children $741.60 of free meal support, so the value is 1."
+us,scenario_117,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,It concluded that free support was $0 from the 130% poverty threshold alone. It ignored Arkansas's universal free breakfast for public school students from 2025-26 (A.C.A. § 6-18-722(f)). That breakfast is worth $741.60 for the two school-age children.
+us,scenario_117,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,"It applied only the federal free-meal income threshold. It missed Arkansas Act 123 of 2025, which makes one breakfast a day free for every public school student regardless of income. That breakfast gives the two K-12 children $741.60 of free support, so the answer is 1."
+us,scenario_117,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"It took high wages and no categorical indicators to mean zero free meal support. It overlooked Arkansas's 2025-26 universal free breakfast for public school students, which ignores income. Under that law, the 16- and 11-year-old get $741.60 of free breakfast support."
+us,scenario_117,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"It correctly found no categorical eligibility and income far above the free limit. It never applied Arkansas Act 123 of 2025, which gives each public school student a free daily breakfast regardless of household income. That breakfast gives the two school-age children $741.60 of free meal support."
+us,scenario_117,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"It treated the income threshold as the only free-meal route. It missed Arkansas's universal no-cost breakfast for public school students under A.C.A. § 6-18-722(f). That breakfast puts breakfast in the free tier for the two K-12 children, worth $741.60, so the value is 1."
+us,scenario_117,free_school_meals_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,"It correctly ruled out the income-based and categorical federal routes, but it missed Arkansas's 2025-26 free breakfast for every public school student (Act 123 of 2025). That breakfast is a state rule outside both federal routes. It gives the 16- and 11-year-old $741.60 of free support."
+us,scenario_117,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,It stopped at the federal free-meal income threshold. It did not apply Arkansas Act 123 of 2025's universal free breakfast for public school students. That breakfast makes free school meal support positive ($741.60 for the two K-12 children).
+us,scenario_117,free_school_meals_eligible,gpt-6-astra,llm_error,state_local_rule,False,"It limited itself to the income-tested 130% FPL rule and federal categorical eligibility. It ignored Arkansas's state rule, A.C.A. § 6-18-722(f), which gives every public school student without another free breakfast one free breakfast a day. That rule yields $741.60 of free support for the two school-age children."
+us,scenario_117,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"It applied only the federal free-meal income limit. It missed Arkansas's 2025-26 universal free breakfast, which applies regardless of income. That breakfast gives the household's two K-12 children $741.60 of free meal support, so the answer is 1."
+us,scenario_117,free_school_meals_eligible,gpt-6-sol,llm_error,state_local_rule,False,"It took high income to mean no free school meal support. Arkansas Act 123 of 2025 gives every public school student one free breakfast a day at any income, and that is worth $741.60 for the 16- and 11-year-old."
+us,scenario_117,free_school_meals_eligible,gpt-6.1-sol,llm_error,state_local_rule,False,"It correctly ruled out income-based free meals and categorical eligibility, but it missed Arkansas's universal free breakfast for public school students from 2025-26 (A.C.A. § 6-18-722(f)). That breakfast gives the two school-age children $741.60 of free breakfast support, so the value is 1."
+us,scenario_117,free_school_meals_eligible,gpt-6.1-terra,llm_error,state_local_rule,False,"It treated the income threshold and categorical qualification as the only routes to free meals. It ignored Arkansas Act 123 of 2025, which provides a free daily breakfast to every public school student. That breakfast is $741.60 of free support for the two K-12 children."
+us,scenario_117,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,It took high income to rule out free meals. It overlooked Arkansas's 2025-26 law giving each public school student a no-cost breakfast regardless of income. That law gives the 16- and 11-year-old $741.60 of free breakfast support.
+us,scenario_117,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"It concluded PolicyEngine returns $0 from the 130% FPL limit alone. It missed that PolicyEngine applies Arkansas Act 123 of 2025's universal free breakfast, which values the two K-12 children's breakfast at $741.60 ($2.06 a day × 180 days × 2). The output is therefore 1."
+us,scenario_117,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"It correctly found income above 130% FPL and no SNAP/TANF/Medicaid categorical eligibility. It never applied Arkansas's universal free breakfast under A.C.A. § 6-18-722(f), which ignores income. That breakfast gives the two school-age children $741.60 of free support."
+us,scenario_117,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"It treated the income and categorical tests as decisive. It missed Arkansas Act 123 of 2025, which gives every public school student one free breakfast a day from 2025-26. That breakfast puts breakfast in the free tier for the 16- and 11-year-old, worth $741.60, so the value is 1."
+us,scenario_117,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"It applied only the 130% FPL free-meal threshold. It ignored Arkansas's 2025-26 universal free breakfast for public school students, which gives the two K-12 children $741.60 of free meal support at any income."
+us,scenario_117,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,It said outright that Arkansas has no universal free school meals regardless of income. That is wrong for breakfast: Act 123 of 2025 (A.C.A. § 6-18-722(f)) gives every public school student one free breakfast a day from 2025-26. That breakfast gives the two K-12 children $741.60 of free support.
+us,scenario_117,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"It correctly found that the $248,675 in wages far exceeds 130% of the five-person guideline and that there is no categorical eligibility. It missed Arkansas Act 123 of 2025, which provides one free breakfast a day to every public school student regardless of income. That breakfast gives the 16- and 11-year-old $741.60 of free support, so the answer is 1."
+us,scenario_117,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"It stopped at the 130% FPL free-meal limit. It did not apply Arkansas's universal free breakfast for public school students (A.C.A. § 6-18-722(f)), which gives the two school-age children $741.60 of free breakfast support."
+us,scenario_117,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,It treated the income threshold as the only route to free meals. It missed Arkansas's 2025-26 law giving each public school student a free daily breakfast regardless of income. That law gives the two K-12 children $741.60 of free support.
+us,scenario_117,free_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,"It ruled out the federal income and direct-certification routes, but it never applied Arkansas Act 123 of 2025. That law gives every public school student a free breakfast a day regardless of income. The household's two school-age children therefore receive $741.60 of free breakfast support, so the value is 1."
+us,scenario_117,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"It judged positive free-meal support only by the income limit for five people. It overlooked Arkansas's universal free breakfast under A.C.A. § 6-18-722(f), which gives the 16- and 11-year-old $741.60 of free breakfast at any income."
+us,scenario_117,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"It recognized only the 130% poverty test and SNAP/TANF participation as routes to free meals. It missed Arkansas Act 123 of 2025, which provides every public school student one free breakfast a day. PolicyEngine values that breakfast at $741.60 for the two K-12 children."
+us,scenario_117,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"It compared the $248,675 in wages with USDA free-meal income guidelines and stopped there. It ignored Arkansas's 2025-26 universal free breakfast for public school students, which makes breakfast free for the two school-age children ($741.60 a year) regardless of income."
+us,scenario_117,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,"It applied only the income-based free-meal limit. It missed Arkansas Act 123 of 2025's free daily breakfast for every public school student, which gives the 16- and 11-year-old $741.60 of free support. The answer is therefore 1."
us,scenario_117,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a household-level “young-child context” pathway and transferred the two-year-old child's WIC category to the 44-year-old head. The head has no listed pregnancy, postpartum, or breastfeeding status and is neither an infant nor a child under five; independently, household income exceeds the 185% poverty limit."
us,scenario_117,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model computed the correct $11,439 SS tax at the $184,500 base, but then switched to a $183,600 base ($14,988.99). It submitted $12,896.44, which matches none of its own calculations. Its final value is disconnected from its correct reasoning."
us,scenario_117,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model added the $35,525 FLSA overtime premium on top of gross wages that already include overtime. It used the 2024 SS wage base of $168,600 instead of $184,500 and applied Additional Medicare Tax at the $200,000 single threshold instead of the $250,000 MFJ threshold. It also counted Arkansas income tax withholding as a payroll tax, then added an invented $1,680 'adjustment'."
@@ -10007,112 +10116,114 @@ us,scenario_117,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The m
us,scenario_117,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly computed SS tax of $11,439 at the $184,500 base, Medicare of $3,606 and $0 Additional Medicare Tax. Those sum to $15,045, but it submitted $20,603, a number disconnected from its own components."
us,scenario_117,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_117,spouse_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly transferred a young child's WIC categorical status to the spouse. Living with a WIC-age child does not make an adult eligible; the spouse has no stated pregnancy, postpartum, or breastfeeding status and the household also fails WIC's income test."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It used the roughly $2,340 standard deduction instead of itemizing the $24,646.33 charitable deduction, so its taxable income came out near $245,805 rather than $223,498.20. It then cut its own $9,196 figure to $8,081 through unexplained 'bracket smoothing'. It also never applied the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"Its AGI of $248,145 was right, but it deducted the full $25,900 charitable gift instead of the reduced $24,646.33 Arkansas itemized total, giving taxable income of $222,245. It then used a 2%/4%/3.9% schedule that produced $8,586.66, where the engine's 2026 schedule gives $8,189.53. Its credits were $145, missing the $60 additional credit for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $35,525 FLSA overtime premium on top of gross wages, even though wages already include it. It also counted the state refund and subtracted a made-up $18,398 of self-employment tax. It then took the standard deduction instead of itemizing, taxed income above $9,799 at a nonexistent 5% rate, and invented $1,050 of credits; the actual credits are $205."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It explicitly skipped the Arkansas calculation and entered 0. The household has $223,498.20 of Arkansas taxable income, which gives $8,189.53 of tax less $205 of credits, or $7,984.53."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It imputed about $22,500 of mortgage interest from the $501,000 balance, but no interest was paid in the inputs, so that deduction is $0. Its own arithmetic came to about $7,645, yet it submitted an unsupported $11,264. It also skipped the itemized reduction and the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It counted the $2,589 state refund in Arkansas income and imputed about $20,040 of mortgage interest that the inputs don't support, giving roughly $202,000 of taxable income. It then stated a tax of $11,820, which is far above what 3.9% of its own base yields."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It added mortgage interest and medical deductions that don't exist: there is no interest-paid input, and medical costs of $11,823 are below the AGI floor. That pushed taxable income down to about $190,000 instead of $223,498.20. It also subtracted only $145 of credits instead of $205."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Its taxable income of $224,834 keeps the $2,589 state refund, which Arkansas excludes, and deducts the full $25,900 instead of $24,646.33. It then used the 3.9% top-rate schedule to get $8,685 rather than the engine's 2026 schedule, and subtracted only $145 of credits, missing the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It wrongly treated the $35,525 overtime premium as exempt from Arkansas tax and kept the state refund in income. It also added a $10,000 SALT deduction with no state or local taxes in the inputs and claimed a $240 Arkansas child-care credit. That credit is $0 because the head has no earnings, so the federal CDCC is $0. All of this pushed its taxable income down to $178,861, against $223,498.20 in the reference."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It imputed $20,040 of mortgage interest even though no interest was paid in the inputs, and skipped the personal credits. It reached about $8,086 and then raised the figure to $12,500 without any computation."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used the Arkansas standard deduction on about $245,000 of income instead of itemizing $24,646.33 of charitable gifts, which overstates taxable income by about $21,500. It then taxed that base at 3.9% instead of the engine's lower 2026 schedule."
-us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used an out-of-date 4.9% top rate on about $184k of taxable income, which understates the base; the correct figure is $223,498.20 after the $24,646.33 itemized deduction. It also claimed no nonrefundable credits apply, leaving out the $145 personal credits and the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It took a $5,000 standard deduction instead of itemizing the $24,646.33 charitable deduction, giving $243,145 of taxable income. It then applied a 3.9% schedule and subtracted none of the $205 in nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its $185,403.68 of taxable income plus $29,805.33 of itemized deductions adds up to $215,209, which means it subtracted the $35,525 overtime premium and kept the state refund. The engine's Arkansas AGI is $248,144.53, with no overtime exclusion and the refund excluded. Its itemized total was also too high: $29,805.33 against $24,646.33."
-us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It taxed $246,334 at an out-of-date 2%/4%/4.4% schedule. That base keeps the state refund and uses a $4,400 standard deduction instead of the $24,646.33 itemized deduction. It also subtracted none of the $205 in nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It started from $250,734, which includes the $2,589 state refund that Arkansas excludes. It also itemized $26,641, adding $741 of medical costs that fall below the floor, instead of $24,646.33. On top of that it used the 3.9% schedule, which gave $8,413.63 instead of the engine's $8,189.53, and subtracted only $145 instead of $205."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no derivation. Its $10,560 matches taxing nearly all of AGI at about a 4.3–4.4% rate after only a standard deduction. The correct steps are itemizing $24,646.33, applying the engine's 2026 schedule ($8,189.53) and subtracting $205 of credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It gave no derivation. Its $8,768 matches applying the 3.9% schedule to roughly $225k–$230k of taxable income with little or no credit offset. The engine applies its 2026 schedule to $223,498.20 to get $8,189.53, then subtracts $205 of credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It started from $250,734, keeping the $2,589 state refund that Arkansas excludes, and applied the 3.9% bracket structure to the full charitable deduction. That overstated the tax relative to the engine's $8,189.53 on $223,498.20, and it subtracted less than the $205 of credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation. Its $10,879 matches taxing nearly all of AGI at an out-of-date 4.4% rate with only a standard deduction. The engine itemizes $24,646.33, applies its 2026 schedule to $223,498.20, and subtracts $205 of credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It gave no derivation. Its $8,234.12 is $249.59 too high, which matches using the 3.9% bracket schedule instead of the engine's 2026 schedule ($8,189.53 on $223,498.20) and not subtracting the full $205 of credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $9,260 matches taxing about $245,000 at 3.9% after only the standard deduction, which skips the $24,646.33 charitable itemized deduction and the engine's lower 2026 schedule."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It explicitly took the Arkansas standard deduction instead of itemizing the $24,646.33 charitable deduction, and taxed the result at 3.9%. That overstates taxable income by about $21,500 and uses a higher rate than the engine's 2026 schedule."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"It deducted the full $25,900 charitable gift instead of the reduced $24,646.33 Arkansas itemized total. It then used the 3.9% schedule instead of the engine's 2026 schedule, which gives $8,189.53, and subtracted only $145 of personal credits, leaving out the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"Its AGI of $248,145 was right, but it deducted the full $25,900 and taxed $222,245 using an out-of-date schedule with 3.4%, 4.7% and 4.4% brackets. It also subtracted none of the $205 in nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It kept the $2,589 state refund in Arkansas AGI and added a medical deduction, even though medical costs are below the floor. It also said no Arkansas nonrefundable credits apply, so it never subtracted the $145 personal credits or the $60 additional credit, and it rounded loosely to $8,000 instead of computing the brackets."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It called the tax 'modest' at $1,005, which is far too low for $223,498.20 of Arkansas taxable income. That income produces $8,189.53 of bracket tax before the $205 of credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It claimed deductions wipe out Arkansas tax. The only deduction the household can take is $24,646.33 of itemized charitable gifts, which leaves $223,498.20 taxable and $7,984.53 of tax after credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It deducted the full $25,900 instead of the reduced $24,646.33 Arkansas itemized total, giving $222,245 of taxable income. Its high-income schedule produced $8,496.66 instead of the engine's $8,189.53, and its $160 of credits fell short of the correct $205 ($145 personal plus $60 additional)."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It took only standard and personal deductions instead of itemizing the $24,646.33 charitable deduction, and said no nonrefundable credit applies. That ignores the $145 personal credits and the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Its $8,107 is $122.47 too high. That matches a bracket tax of about $8,252, roughly $62 above the engine's $8,189.53 on $223,498.20, less only the $145 personal credits, with the $60 additional credit left out."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $7,995.31 is $10.78 too high. That matches deducting the full $25,900 charitable gift instead of the reduced $24,646.33 total, and claiming only the $145 personal and dependent credits without the $60 additional credit; the two errors nearly cancel out."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It correctly left out the refund but deducted the full $25,900, giving $222,245 of taxable income, instead of the reduced $24,646.33. It then applied the 3.9% high-income schedule instead of the engine's 2026 schedule, which yields $8,189.53, and subtracted only $145, missing the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It gave no breakdown. Its $9,535 matches taxing almost all of the roughly $248k AGI at 3.9% without itemizing the $24,646.33 charitable deduction. That overstates taxable income by about $24,600 compared with $223,498.20."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It applied a 3.9% top marginal rate to income after the full charitable deduction. The engine's 2026 schedule yields only $8,189.53 on $223,498.20 (about 3.66% effective), and the engine also subtracts the $60 additional credit on top of the $145 personal credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"It deducted the full $25,900 from $248,145 instead of the reduced $24,646.33 itemized total. It then used the 3.9% high-income schedule rather than the engine's 2026 schedule, which gives $8,189.53, and subtracted only $145, missing the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It used the joint standard deduction instead of itemizing the $24,646.33 charitable deduction, which overstates taxable income by more than $20,000 compared with $223,498.20."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It submitted a round $8,000 without working through the brackets or credits. The engine gets $7,984.53 by taxing $223,498.20 at its 2026 schedule ($8,189.53) and then subtracting $205 of nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied an out-of-date top rate of about 4.4% to roughly $224,000 of taxable income. The engine's 2026 schedule yields $8,189.53 on $223,498.20, about 3.66% effective, and the engine also subtracts $205 of nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It started from $250,734, keeping the $2,589 state refund that Arkansas excludes, and added a $741 medical deduction for costs below the floor. It used the 3.9% schedule instead of the engine's 2026 schedule and said no Arkansas nonrefundable credits apply, leaving out the $145 personal credits and the $60 additional credit."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It started from $223,734, about $24,410 below the engine's Arkansas AGI of $248,144.53. Arkansas AGI includes all wages, overtime premium included, minus only the $530 capital loss. Deducting the full $25,900 from that too-low figure understated taxable income at $197,834."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It treated Arkansas as a flat 3.9% tax with no bracket structure, starting from a wrong AGI of $242,511, and deducted the full $25,900. It also subtracted none of the $205 in nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It reached about $221,000 of taxable income through 'standard deduction/exemptions' instead of the $24,646.33 itemized deduction. It then taxed that at a flat 3.9% instead of the engine's 2026 schedule, which gives $8,189.53 on $223,498.20, and subtracted only small credits rather than $205."
-us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value and no explanation for state_income_tax_before_refundable_credits, so there was nothing to score against the $7,984.53 reference."
-us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It took the joint standard deduction, giving $245,654 of taxable income, instead of itemizing the $24,646.33 charitable deduction. It then used the 3.9% high-income schedule and subtracted only $145 of credits instead of $205."
-us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported no Arkansas liability at all. With $223,498.20 of taxable income after the $24,646.33 itemized deduction, the 2026 schedule gives $8,189.53, and subtracting $205 of credits leaves $7,984.53."
-us,scenario_117,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It counted medical costs as itemized deductions even though they fall below the floor. It also used the full charitable gift instead of the reduced $24,646.33 total, taxed the result using a 3.9% top rate instead of the engine's 2026 schedule, and never subtracted the $205 of nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It took the Arkansas joint standard deduction, giving about $243,435 of taxable income, instead of itemizing the $24,646.33 charitable deduction. It then applied 3.9% graduated rates and subtracted only $145 of credits instead of $205."
-us,scenario_117,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It kept the $2,589 state refund in Arkansas AGI ($250,734) and deducted the full $25,900, giving $224,834 of taxable income instead of $223,498.20. It also guessed credits of about $155 (roughly $31 per person) instead of the $205 total of $145 personal plus $60 additional."
-us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $35,525 overtime premium on top of $248,675 of gross wages, even though wages already include it, and counted the state refund, giving $286,259. It then dropped its own itemized calculation, used the $4,400 standard deduction, and applied a flat 4.4% rate."
-us,scenario_117,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from $213,150 and subtracted a made-up $28,867 'filing deduction' instead of starting from $248,144.53 and taking the $24,646.33 itemized deduction. It claimed a $60 credit of $20 per dependent, below the engine's $205 credit total, and its submitted $5,105 contradicts its own arithmetic result of $7,127."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It took a roughly $2,340 standard deduction instead of itemizing the $24,646.33 charitable deduction, so its taxable income was about $245,805 instead of $223,498.20. It then cut its own $9,196 result to $8,081 with an unexplained 'bracket smoothing' step. It also missed the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It deducted the full $25,900 of charity without the 0.5%-of-AGI floor, so its taxable income was $222,245 instead of $223,498.20. Its 2%/4%/3.9% table gave $8,586.66 of tax, while the engine's 2026 schedule gives $8,189.53. It subtracted only $145 of credits and missed the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,thresholds_rates,False,"It got the floored charitable deduction right ($24,646.33) but started from $250,734, which wrongly includes the $2,589 state tax refund that Arkansas excludes. Its 2%/3.9% table then gave $8,731.92 instead of the engine's $8,189.53 on $223,498.20. It subtracted only $145 and missed the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It added the $35,525 overtime on top of the $248,675 wages, even though the wages already include it, and subtracted made-up items such as self-employment tax. It also skipped the charitable itemized deduction and used an outdated 5% top rate. Its $1,050 of credits is invented; the actual credits are $205."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,missing_output,False,"It entered $0 without computing anything. Arkansas taxable income of $223,498.20 produces $8,189.53 of tax, which falls to $7,984.53 after $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It made up about $22,500 of mortgage interest from the $501,000 balance; unlisted interest is $0, so only the $24,646.33 floored charitable deduction applies. It then submitted $11,264, which does not match its own estimate of about $7,645, and it missed the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It kept the $2,589 state tax refund in Arkansas income and made up about $20,040 of mortgage interest from the loan balance. The $11,820 it reported does not follow from its own $202,000 taxable income at 3.9%, and its $145 of credits leaves out the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It added imputed mortgage interest and a medical deduction, cutting taxable income to about $190,000. The only allowable itemized deduction is the $24,646.33 floored charitable deduction, which leaves $223,498.20 taxable. It subtracted only $145 of credits instead of $205."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"Its net income of $224,834 includes the excluded $2,589 state refund and subtracts the full $25,900 of charity without the 0.5%-of-AGI floor. Its 3.9% high-income schedule gave $8,685 of tax, compared with the engine's $8,189.53. It subtracted only $145 of credits instead of $205."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It removed the $35,525 overtime from Arkansas income; the engine's Arkansas AGI of $248,144.53 includes it. It also added a $10,000 SALT deduction for taxes that were never listed, a $448 employee-expense deduction, and a $240 Arkansas child care credit. That credit is zero because the head has no earnings. The result is taxable income of $178,861 instead of $223,498.20, and the $60 additional credit is missing."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It made up about $20,040 of mortgage interest from the loan balance when unlisted interest is $0. It then raised its own computed figure of about $8,086 to $12,500 with no supporting calculation, and it applied no personal or additional credits."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It used the Arkansas standard deduction, leaving about $245,000 taxable, instead of itemizing the $24,646.33 charitable deduction that brings taxable income to $223,498.20. Its 3.9% approximation also overstates the engine's $8,189.53 tax before the $205 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,thresholds_rates,False,"It used an outdated 4.9% top rate on about $184,000 of taxable income instead of the 2026 schedule on $223,498.20. It also stated that no nonrefundable credits apply, leaving out the $145 personal credits and the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It took a $5,000 standard deduction instead of itemizing the $24,646.33 charitable deduction, so its taxable income was $243,145 instead of $223,498.20. It applied no nonrefundable credits, leaving out the $145 personal credits and the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"Its $185,403.68 of taxable income comes from removing the $35,525 overtime from Arkansas income (the engine's AGI includes it) and claiming $29,805.33 of itemized deductions. Only the $24,646.33 charitable deduction is allowable. That understated taxable income by about $38,000 against $223,498.20."
+us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,thresholds_rates,False,"It used a pre-2024 2%/4%/4.4% table instead of the 2026 schedule. It also kept the $2,589 state refund in income and took a $4,400 standard deduction instead of the $24,646.33 charitable deduction. It subtracted none of the $205 of personal and additional credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It started from $250,734, which includes the excluded state refund. It deducted the full $25,900 of charity plus $741 of medical costs that fall below the 7.5%-of-AGI floor. Its 3.9% schedule gave $8,413.63 of tax, compared with the engine's $8,189.53, and its $145 of credits leaves out the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It gave no computation. $10,560 matches an outdated higher rate applied without the charitable itemized deduction. The engine taxes $223,498.20 at $8,189.53 and then subtracts $205 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It gave no computation. $8,768 matches the 3.9%-top-rate table applied to an unfloored or refund-inflated taxable income of about $227,000 with only $145 of credits. The engine produces $8,189.53 on $223,498.20 and then subtracts $205 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It started from $250,734 AGI, which includes the $2,589 state refund that Arkansas excludes, and deducted the full $25,900 of charity without the 0.5%-of-AGI floor. It used a 3.9%-based schedule that overstates the engine's $8,189.53 tax, and it left out the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no computation. $10,879 matches a standard-deduction calculation at an outdated rate of about 4.4%. The engine itemizes the $24,646.33 charitable deduction, taxes $223,498.20 at $8,189.53, and subtracts $205 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It gave no computation. Its $8,234.12 is $249.59 too high, which matches subtracting only the $145 personal credits (missing the $60 additional credit) from a pre-credit tax about $190 above the engine's $8,189.53 on $223,498.20."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no computation. $9,260 matches applying the 3.9% table after only the standard deduction. The engine itemizes the $24,646.33 charitable deduction, which reduces taxable income to $223,498.20."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It explicitly used the Arkansas standard deduction instead of itemizing the $24,646.33 charitable deduction, which overstated taxable income by more than $20,000 against $223,498.20. It also applied the 3.9% top rate instead of the engine's schedule, which gives $8,189.53."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,thresholds_rates,False,"It deducted the full $25,900 of charity instead of the $24,646.33 left after the 0.5%-of-AGI floor. Its 3.9% schedule gave about $8,698 before credits, compared with the engine's $8,189.53. It subtracted only $145 and missed the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used an outdated regular-table schedule with 4.7% and 4.4% brackets instead of the 2026 schedule. It deducted the full $25,900 without the 0.5%-of-AGI floor. It also applied none of the $205 of personal and additional credits."
+us,scenario_117,state_income_tax_before_refundable_credits,glm-5.3,llm_error,other,False,"It stated that no Arkansas nonrefundable credits apply, leaving out the $145 personal credits for five people and the $60 additional credit. It also kept the excluded $2,589 state refund in AGI and added a medical deduction below the 7.5% floor."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"It treated the Arkansas tax as 'modest' at $1,005. Taxable income of $223,498.20 produces $8,189.53 under the 2026 schedule, so the model understated the liability by roughly $7,000 before the $205 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It reported $0, citing unspecified high deductions. The only itemized deduction is $24,646.33 of charity, which leaves $223,498.20 taxable and $7,984.53 of tax after $205 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It deducted the full $25,900 of charity without the 0.5%-of-AGI floor, giving $222,245 of taxable income. Its high-income schedule gave $8,496.66 instead of the engine's $8,189.53. It subtracted $160 of credits instead of the correct $205 ($145 personal plus the $60 additional credit)."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used standard and personal deductions instead of itemizing the $24,646.33 charitable deduction. It also stated that no nonrefundable credit applies, leaving out the $145 personal credits and the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"It itemized the charity and subtracted personal credits, but it is $122 too high. That gap equals the missing $60 additional credit plus a pre-credit tax about $62 above the engine's $8,189.53 on $223,498.20."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"It came within $10.78 of the reference but did not reproduce the engine's calculation exactly. That calculation is $8,189.53 of tax on $223,498.20 (after the $24,646.33 floored charitable deduction), less $145 of personal credits and the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,thresholds_rates,False,"It deducted the full $25,900 of charity, giving $222,245 of taxable income, and applied the 3.9%-top-rate high-income table, which overstates the engine's $8,189.53 tax. It subtracted only $145 and missed the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It gave no computation. $9,535 matches taxing roughly the full $248,000 of wages after only a standard deduction. The engine itemizes the $24,646.33 charitable deduction, which reduces taxable income to $223,498.20."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,thresholds_rates,False,"It applied a 3.9% top-rate table after the full unfloored $25,900 charitable deduction and subtracted only the personal credits. The engine's 2026 schedule gives $8,189.53 on $223,498.20, and $205 of credits (including the $60 additional credit) brings it to $7,984.53."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,thresholds_rates,False,"It used $248,145 less the full $25,900 of charity (no 0.5%-of-AGI floor) and the 3.9% high-income schedule, which overstates tax by about $400 against the engine's $8,189.53. It subtracted $145 and missed the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It explicitly took the joint standard deduction instead of itemizing the $24,646.33 charitable deduction, which overstated taxable income by more than $20,000 against $223,498.20."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It submitted a rounded $8,000 guess without computing taxable income, the rate schedule, or credits. The correct steps are $223,498.20 of taxable income, $8,189.53 of tax, and $205 of credits, which gives $7,984.53."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied an outdated top rate of about 4.4% to about $224,000 of taxable income instead of the 2026 schedule, which gives $8,189.53 on $223,498.20. It also did not subtract the $205 of personal and additional credits."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It started from $250,734, which includes the excluded state refund, and deducted $741 of medical costs below the 7.5% floor plus the full $25,900 of charity. It also stated that no nonrefundable credits apply, leaving out the $145 personal credits and the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It understated federal AGI as $223,734 instead of $250,734, which led to $197,834 of taxable income instead of $223,498.20. It also deducted the full $25,900 of charity and subtracted only $145 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted the head's $8,223 employer-plan health premium as an above-the-line deduction, which made AGI $242,511. It applied a flat 3.9% with no lower brackets and subtracted none of the $205 of personal and additional credits."
+us,scenario_117,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied about 3.9% to roughly $221,000 of taxable income and subtracted only about $100 of credits. The engine's schedule gives $8,189.53 on $223,498.20, and $205 of credits (the $145 personal credits plus the $60 additional credit) brings it to $7,984.53."
+us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no usable value for state_income_tax_before_refundable_credits, so there was nothing to score against the $7,984.53 reference."
+us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It took the joint standard deduction, leaving $245,654 taxable, instead of itemizing the $24,646.33 charitable deduction that brings taxable income to $223,498.20. It subtracted only $145 of credits and missed the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It reported no Arkansas liability, even though $223,498.20 of taxable income produces $8,189.53 of tax, or $7,984.53 after $205 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,thresholds_rates,False,"It deducted medical costs below the 7.5% floor along with the full $25,900 of charity, and it applied a flat 3.9% approximation. It also miscalculated 3.9% of $221,545 as $8,340 instead of about $8,640. It never subtracted the $205 of credits."
+us,scenario_117,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It took the Arkansas joint standard deduction, leaving about $243,435 taxable, instead of itemizing the $24,646.33 charitable deduction that brings taxable income to $223,498.20. It subtracted only $145 and missed the $60 additional credit."
+us,scenario_117,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"It kept the $2,589 state refund in Arkansas AGI ($250,734) and deducted the full $25,900 of charity without the 0.5%-of-AGI floor. It approximated the rate schedule at about 3.8% and subtracted $155 of credits instead of $205 ($145 personal plus $60 additional)."
+us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It added the $35,525 overtime on top of wages that already include it and kept the state refund, inflating AGI to $286,259. It then dropped its own itemized calculation in favor of a $4,400 standard deduction at an outdated flat 4.4% rate."
+us,scenario_117,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It started from an unexplained $213,150 of income and subtracted a made-up $28,867 'standard/filing deduction', giving $184,283 instead of $223,498.20. It left out the $145 personal credits, and it submitted $5,105 even though its own arithmetic gave $7,127."
us,scenario_117,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_118,head_medicaid_eligible,claude-haiku-5.5,llm_error,categorical_eligibility,False,"The model said Social Security income is modest but still concluded 'no Medicaid enrollment assumed; not eligible'. That confuses enrollment with eligibility, and Medicare eligibility does not rule out Medicaid. It also missed that $2,800 a year in Social Security and $70 in assets qualify the head for SSI ($9,368 a year), and in NY that gives automatic Medicaid through the SSI-recipient category."
us,scenario_118,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model said the facts were not enough to support eligibility, even though they fully establish the aged/blind/disabled pathway. The head is 74, blind, and disabled, with $2,800 a year in Social Security and $70 in assets. That qualifies them for SSI, which gives automatic NY Medicaid through the SSI-recipient category, and they also pass NY's aged/disabled income and asset tests. The model never applied these tests to the listed income and resources."
us,scenario_118,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that the $2,800 Social Security retirement benefit is not subject to employee payroll tax and that no wages were provided, but then contradicted that reasoning by assigning $172.90 of employee-side tax. It improperly placed retirement benefits or another nonexistent amount in the payroll-tax base instead of applying the tax rates to a zero earned-income base."
-us,scenario_118,snap,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only the $233/month of Social Security and left out the $780.67/month of SSI the blind, disabled 74-year-old receives, which is countable SNAP unearned income. That pushed net income to $0 and the answer to the maximum allotment. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI correctly, but its shelter costs were only the $599 of mortgage interest and property tax. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly concluded the household is over the SNAP income limit. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 without a budget calculation. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It treated $233/month of Social Security as the only income and missed the $780.67/month of SSI that this aged, blind, disabled, low-asset head receives. It also left out the utility allowance from shelter costs. Leaving out SSI drove its calculated net income to $0. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"It left out the $780.67/month of SSI and ignored the listed mortgage interest and property tax as shelter costs, so its net income was only $29. It then submitted $2,533, which doesn't follow from its own ~$283/month calculation. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, left out the $780.67/month of SSI, and skipped the excess shelter deduction. Its arithmetic also fails: $298 − 0.3 × $29 is $289, not $203. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It used $599/month of mortgage interest and property tax as shelter costs. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly assumed no SSI, but the blind, disabled 74-year-old with $70 of assets receives $780.67/month of SSI, and that counts as SNAP income. Without it, net income went to $0 and the answer to a $292 maximum. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security and left out the $780.67/month of SSI. It then took an arbitrary cut from a $3,504 maximum instead of computing 30% of net income. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It treated income as $233/month and missed the $780.67/month of SSI the head receives, which is countable SNAP unearned income. That drove net income to $0 and the answer to the $298 maximum. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, left out the $780.67/month of SSI, and ignored the mortgage interest and property tax shelter costs altogether. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"It never counted the $780.67/month of SSI as income, so its deductions wiped out net income and it returned the $292 FY2025 maximum. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI and used $598.75 of mortgage interest and property tax as shelter costs and a $303 maximum allotment. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It explicitly treated SSI as excluded from SNAP income, but SSI is countable unearned income for SNAP. Leaving out the $780.67/month drove net income to $0 and the answer to a $299 maximum. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated SSI at $864.67/month and net income at $0. The current engine counts $780.67 of federal SSI plus $87 of New York SSP, totaling $867.67/month before Social Security. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It said the household's expenses rule out a benefit, which gets the formula backwards: shelter costs increase the deduction. SSI confers categorical eligibility. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"It overstated SSI at $817/month (the correct figure is $780.67) and used a $198 standard deduction and $291 maximum instead of $209 and $298. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It applied New York's NYSNIP standardized benefit and assigned the maximum $291. The reference uses the standard SNAP budget instead: $1,014 of gross income, minus the $209 standard deduction and an uncapped shelter-plus-utility deduction of about $611, gives net income of about $194 and $298 − $58.20 = $239.80/month."
-us,scenario_118,snap,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It treated Social Security as the only income and never counted the $780.67/month of SSI, so its shelter deduction wiped out net income and produced the $291 maximum. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It attributed its $828 annual estimate to standard and excess shelter deductions for mortgage interest and property tax. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"It used 2025-level SSI ($9,044/year), a $200/month standard deduction and a $292 maximum. Its main error was building shelter costs from only the $7,185 of mortgage interest and property tax, without the utility allowance, which gave net income of $6,981/year instead of about $194/month. The reference's larger shelter deduction gives $239.80/month."
-us,scenario_118,snap,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"It based its $843 annual estimate on Social Security, SSI and shelter expenses from mortgage interest and property tax. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only $233.33/month of Social Security and missed the $780.67/month of SSI the head receives, so net income came out at $0 and the answer at a $295 maximum. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, left out the $780.67/month of SSI, and never applied the excess shelter deduction for the listed mortgage interest and property tax. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed there were no expenses or other eligibility facts, even though mortgage interest and property tax are listed and the head qualifies for SSI, which confers categorical SNAP eligibility. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 for lack of supporting facts. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"It applied SSI's $20 general income exclusion to SNAP, left out the $780.67/month of SSI income, and used a $243 maximum allotment instead of the $298 FY2026 figure. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only $2,800 of Social Security and left out the $9,368 of federal SSI the head receives, so it landed near the maximum allotment. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It estimated $115/month after the standard and excess shelter deductions. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI and used the mortgage interest and property tax as shelter costs, but left out the utility allowance. That kept the excess shelter deduction near $200 and net income near $600, giving $123/month. The reference's shelter deduction of about $611 gives net income of about $194 and $239.80/month."
-us,scenario_118,snap,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"It assumed no shelter deduction even though $5,551 of mortgage interest and $1,634 of property tax are listed, so it taxed the full $805 of adjusted income. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"It treated income as very low and gave the $298 maximum, which means it left out the $780.67/month of SSI the head receives. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It set SSI at $10,412/year by adding a New York state supplement to the $9,368 federal amount, and its $1,833 excess shelter deduction used only the mortgage interest and property tax, without the utility allowance. The current engine counts the New York supplement in SNAP income. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI and used the right $298 maximum and $209 standard deduction, but built shelter costs from only the mortgage interest and property tax. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"It applied no shelter deduction. The engine counts the listed property tax in housing_cost; whether the listed mortgage interest belongs to the occupied home remains unresolved. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"It returned $0 without doing any calculation. SSI confers categorical eligibility. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Its net income of about $735 comes from adding a New York state supplement to SSI and using only the mortgage interest and property tax as shelter costs, without the utility allowance. It also used a $301 maximum instead of $298. The reference counts $1,014 of gross income and an excess shelter deduction of about $611, which gives net income of about $194 and $239.80/month."
-us,scenario_118,snap,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"It got net income to $0 because it left out the $780.67/month of SSI as countable income, and it used the $292 FY2025 maximum. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI and used the right $298 maximum and $209 standard deduction, but its uncapped shelter deduction rested on only the $7,185 of mortgage interest and property tax. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"It said shelter costs were unlisted, ignoring the listed $5,551 of mortgage interest and $1,634 of property tax, and it left out the $780.67/month of SSI. It also used 2024 deduction and allotment values. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"It treated income as $233/month and missed the $780.67/month of SSI, so net income came out at $0 and the answer at the $298 maximum. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,kimi-k2.6,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security and left out the $780.67/month of SSI, which drove net income to $0. It also used an inflated $309 maximum instead of $298. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Its $1,101 gross income correctly includes $10,412/year of federal SSI and New York SSP. It applied the $209 standard deduction and no excess shelter deduction. The 2.60.0 default-county recheck records $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"It claimed the income was too low to qualify and that the formula yields zero, which gets SNAP's income test backwards. Low income increases the calculated benefit. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"It took only the standard deduction and applied no excess shelter deduction for the listed mortgage interest and property tax. Its $1,101 monthly gross income includes the New York supplement counted by the current engine. The 2.60.0 default-county recheck records $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"It counted SSI correctly, but its $196 excess shelter deduction came from only the $599 of mortgage interest and property tax. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"It used the held $209 standard deduction and $298 maximum allotment. The 2.60.0 engine uses the listed property tax and an $877/month utility allowance under its default county; housing_cost does not include mortgage interest. Including the $87 New York supplement gives an excess shelter deduction of about $567, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"It counted only Social Security, left out the $780.67/month of SSI, and never applied the shelter deduction for the listed mortgage interest and property tax. It also used outdated $198 and $291 values. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
-us,scenario_118,snap,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"It wrongly required a qualifying child for SNAP. SNAP covers single-adult households. The 2.60.0 default-county recheck records $1,101 gross income, $325 net income and $200/month. The frozen $2,903.94 remains excluded from scoring."
+us,scenario_118,snap,claude-fable-5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only the $233.33/month of Social Security and left out the $780.67/month of SSI the blind, disabled 74-year-old receives, which SNAP counts as unearned income. That error drove net income to $0 and the answer to the maximum allotment ($293×12). With $1,014 of gross income and shelter costs that include the SUA, net income is $194.38 and the benefit is $239.80/month."
+us,scenario_118,snap,claude-fable-5.1,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SSI correctly and allowed uncapped shelter costs, but used only mortgage interest plus property tax ($599/month) and left out New York's SUA. Adding the SUA raises shelter costs to about $1,013, the excess shelter deduction to $610.62, and lowers net income from $609 to $194.38. The benefit is $239.80/month, not $115."
+us,scenario_118,snap,claude-fable-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SSI but dropped the $462.58/month of mortgage interest and the NY SUA from shelter costs, treating only property tax as shelter, so it allowed no excess shelter deduction and used net income of $805. Correct shelter costs of about $1,013 give a $610.62 deduction, net income of $194.38 and $239.80/month."
+us,scenario_118,snap,claude-haiku-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Called the household ineligible even though it describes the income as low, and it invented a $170 medical standard. The household is categorically eligible through SSI receipt and passes the net income test ($194.38 against $1,304.17), so it receives $239.80/month."
+us,scenario_118,snap,claude-haiku-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Returned $0 without doing any calculation. The household is SNAP-eligible through SSI. Social Security plus SSI is $1,014/month; after the $209 standard deduction and an uncapped excess shelter deduction (mortgage interest, property tax and NY SUA), net income is $194.38 and the benefit is $239.80/month."
+us,scenario_118,snap,claude-opus-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only $233 of Social Security as income and left out the $780.67/month of SSI the head receives, so the shelter deduction pushed net income below zero and it paid the maximum ($293×12). It also left out the NY SUA. Correct net income is $194.38, which gives $239.80/month."
+us,scenario_118,snap,claude-opus-4.8,prompt_ambiguity,taxable_income_or_deductions,False,"Left out both the $780.67/month of SSI income and the uncapped excess shelter deduction, using only the standard deduction on $233 of Social Security. It then cut its own ~$283/month figure to an arbitrary $2,533. The correct net income of $194.38 gives $239.80/month."
+us,scenario_118,snap,claude-opus-5,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the $780.67/month of SSI income and the excess shelter deduction, and got its own arithmetic wrong ($298 − 0.3×$29 is $289, not $203). The correct calculation uses $1,014 of gross income, a $610.62 shelter deduction and $194.38 of net income, giving $239.80/month."
+us,scenario_118,snap,claude-opus-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SSI and uncapped mortgage interest plus property tax, but left New York's SUA out of shelter costs. With the SUA, shelter costs are about $1,013/month, excess shelter is $610.62 and net income is $194.38 rather than $609, so the benefit is $239.80/month, not $115."
+us,scenario_118,snap,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Assumed outright that the head gets no SSI, but an aged, blind, disabled person with $233/month of Social Security receives $780.67/month of SSI, which counts as SNAP income. It also used the outdated $292 maximum instead of $298. Correct net income is $194.38, giving $239.80/month."
+us,scenario_118,snap,claude-sonnet-5,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the $780.67/month of SSI income, concluded net income was about zero, and then applied an unexplained reduction to reach $2,555. Counting SSI and the full shelter costs including the SUA gives $194.38 of net income and $239.80/month."
+us,scenario_118,snap,claude-sonnet-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Treated $233/month of Social Security as the only income and left out the $780.67/month of SSI, so net income came out at zero and it paid the $298 maximum. With SSI counted, net income is $194.38 and the benefit is $239.80/month."
+us,scenario_118,snap,deepseek-v4-flash-0731,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the $780.67/month of SSI income and the uncapped excess shelter deduction for mortgage interest, property tax and the SUA, applying only a $199 standard deduction to $233 of Social Security. The correct $1,014 of income less $209 and a $610.62 shelter deduction gives $194.38 of net income and $239.80/month."
+us,scenario_118,snap,deepseek-v4-pro,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the $780.67/month of SSI as countable income, so its deductions drove net income to $0, and it used the FY2025 $292 maximum. With SSI counted, net income is $194.38 and the benefit is $298 − $58.20 = $239.80/month."
+us,scenario_118,snap,deepseek-v4-pro-0813,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SSI, but left New York's SUA out of shelter costs, so its excess shelter deduction was only $198.75 instead of $610.62. It also used a $303 maximum and a $210 standard deduction instead of $298 and $209. Correct net income is $194.38, giving $239.80/month."
+us,scenario_118,snap,deepseek-v4.1-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Explicitly excluded SSI from countable income, but SSI is unearned income for SNAP, so countable income is $1,014/month, not $233. With the standard and shelter deductions, net income is $194.38 rather than $0, and the benefit is $239.80/month, not the $299 maximum."
+us,scenario_118,snap,gemini-3-flash-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $864.67 (federal SSI is $780.67) and then claimed net income of $0 without doing the arithmetic. With $1,014 of income, the $209 deduction and about $1,013 of shelter costs including the SUA, the excess shelter deduction is $610.62 and net income is $194.38, giving $239.80/month, not the $292 maximum."
+us,scenario_118,snap,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Returned $0 based on a vague claim that income was insufficient. Low income raises SNAP benefits rather than ruling them out. The household is categorically eligible through SSI, and $194.38 of net income gives $239.80/month."
+us,scenario_118,snap,gemini-3.1-pro-preview,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $817 (it is $780.67), left New York's SUA out of shelter costs (excess shelter of $173 instead of $610.62), and used outdated $198 standard deduction and $291 maximum figures. Correct net income is $194.38, giving $239.80/month."
+us,scenario_118,snap,gemini-3.5-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Gave the maximum benefit by applying New York's NYSNIP standardized benefit for SSI recipients instead of the regular SNAP formula. Under the regular formula, SSI is counted with Social Security as $1,014 of income, and after the standard and uncapped shelter deductions net income is $194.38, so the benefit is $298 − $58.20 = $239.80/month."
+us,scenario_118,snap,gemini-3.5-flash-lite,prompt_ambiguity,taxable_income_or_deductions,False,Returned $0 even though the household is SNAP-eligible through SSI and passes the net income test. Social Security plus SSI less the $209 standard deduction and a $610.62 uncapped shelter deduction leaves $194.38 of net income and a $239.80 monthly benefit.
+us,scenario_118,snap,gemini-3.6-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only Social Security as income, so the housing deductions zeroed out net income and it paid an outdated $291 maximum. Adding the $780.67/month of SSI the head receives gives $194.38 of net income and a $239.80 monthly benefit."
+us,scenario_118,snap,gemini-3.7-flash,prompt_ambiguity,taxable_income_or_deductions,False,"A $69/month benefit implies about $763 of net income, so it allowed almost no excess shelter deduction. Mortgage interest ($462.58) plus property tax ($136.17) plus the NY SUA come to about $1,013 of shelter costs, which gives a $610.62 deduction, $194.38 of net income and $239.80/month."
+us,scenario_118,snap,gemini-3.8-flash,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SSI and mortgage plus tax shelter costs, but left New York's SUA out of shelter costs, so excess shelter was $2,463/year instead of about $7,327. It also used outdated $200/month standard deduction and $292 maximum figures. Correct monthly net income is $194.38, giving $239.80/month."
+us,scenario_118,snap,gemini-4-argon,prompt_ambiguity,taxable_income_or_deductions,False,"A $70/month benefit implies about $760 of net income, so it understated the uncapped shelter deduction by leaving out New York's SUA and most of the housing costs. Correct shelter costs of about $1,013 give a $610.62 deduction, $194.38 of net income and $239.80/month."
+us,scenario_118,snap,glm-5.2,prompt_ambiguity,taxable_income_or_deductions,False,"Treated $233.33 of Social Security as the only income and left out the $780.67/month of SSI, so the shelter deduction zeroed net income and it paid an assumed $295 maximum. With SSI counted, net income is $194.38 and the benefit is $239.80/month."
+us,scenario_118,snap,glm-5.3,prompt_ambiguity,taxable_income_or_deductions,False,"Left out both the $780.67/month of SSI income and the uncapped excess shelter deduction, using only the standard deduction on $233 of Social Security. The correct $1,014 of income with a $610.62 shelter deduction gives $194.38 of net income and $239.80/month."
+us,scenario_118,snap,gpt-5.4-mini,prompt_ambiguity,taxable_income_or_deductions,False,"Said the household had no expenses and returned $0, ignoring the listed mortgage interest and property tax and the head's SSI-based categorical eligibility. The correct calculation gives $194.38 of net income and $239.80/month."
+us,scenario_118,snap,gpt-5.4-nano,prompt_ambiguity,taxable_income_or_deductions,False,"Returned $0, claiming supporting facts were missing. The facts given establish SSI receipt (categorical eligibility), $1,014 of monthly income and uncapped shelter costs, which together give $194.38 of net income and $239.80/month."
+us,scenario_118,snap,gpt-5.5,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the $780.67/month of SSI income, applied the SSI program's $20 general income exclusion (which SNAP does not have), and used a $243 maximum allotment instead of the $298 one-person maximum. Correct net income is $194.38, giving $298 − $58.20 = $239.80/month."
+us,scenario_118,snap,gpt-5.6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Treated the $2,800 of Social Security as the only countable income, leaving out the $9,368 of annual SSI, which gave a near-maximum $290.70/month. With SSI counted and the uncapped shelter deduction including the SUA, net income is $194.38 and the benefit is $239.80/month."
+us,scenario_118,snap,gpt-5.6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SSI and mortgage plus tax shelter costs, but left New York's SUA out of shelter costs, so net income was about $609 instead of $194.38. With the SUA, excess shelter is $610.62 and the benefit is $239.80/month, not $115."
+us,scenario_118,snap,gpt-5.6-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Counted Social Security, SSI, mortgage interest and property tax, but left New York's SUA out of shelter costs, so net income was about $583. With the SUA, the excess shelter deduction is $610.62 and net income is $194.38, so the benefit is $239.80/month, not $123."
+us,scenario_118,snap,gpt-6-astra,prompt_ambiguity,taxable_income_or_deductions,False,"Refused any shelter deduction because mortgage payments were not listed, even though mortgage interest and property tax were both given and SNAP counts them as shelter costs along with the NY SUA. The uncapped $610.62 excess shelter deduction lowers net income from $805 to $194.38, giving $239.80/month."
+us,scenario_118,snap,gpt-6-luna,prompt_ambiguity,taxable_income_or_deductions,False,"Treated income as very low by leaving out the $780.67/month of SSI and so paid the $298 maximum. Counting SSI gives $1,014 of gross income and $194.38 of net income after deductions, so the benefit is $239.80/month."
+us,scenario_118,snap,gpt-6-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $10,412/year by adding a state supplement to the $9,368 of federal SSI, and left New York's SUA out of shelter costs, so excess shelter was $1,833/year. Correct monthly figures are $1,014 of income, a $610.62 excess shelter deduction and $194.38 of net income, giving $239.80/month."
+us,scenario_118,snap,gpt-6.1-sol,prompt_ambiguity,taxable_income_or_deductions,False,"Used the correct $298 maximum, $209 standard deduction and SSI income, but counted only mortgage interest and property tax as shelter costs and left out New York's SUA. That left net income at about $609 instead of $194.38, so the benefit was $115 instead of $239.80/month."
+us,scenario_118,snap,gpt-6.1-terra,prompt_ambiguity,taxable_income_or_deductions,False,"Applied no shelter deduction, saying no shelter expense was specified, even though mortgage interest and property tax were listed and the NY SUA applies. The resulting $610.62 uncapped excess shelter deduction lowers net income from $805 to $194.38, giving $239.80/month."
+us,scenario_118,snap,grok-4.3,prompt_ambiguity,taxable_income_or_deductions,False,"Returned $0, citing missing inputs, without applying the SNAP formula. The household is categorically eligible through SSI, and $1,014 of income less the $209 standard deduction and a $610.62 shelter deduction gives $194.38 of net income and $239.80/month."
+us,scenario_118,snap,grok-4.5,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly identified SSI categorical eligibility, but reached $735 of net income by overstating income and leaving New York's SUA out of the uncapped shelter costs. It also used a $301 maximum. Correct net income is $194.38, giving $298 − $58.20 = $239.80/month."
+us,scenario_118,snap,grok-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"Left the $780.67/month of SSI out of countable income, so the uncapped shelter deduction zeroed net income and it paid the outdated $292 maximum. With SSI counted, net income is $194.38 and the benefit is $239.80/month."
+us,scenario_118,snap,grok-4.7,prompt_ambiguity,taxable_income_or_deductions,False,"Correctly counted $9,368 of federal SSI plus Social Security and allowed uncapped shelter costs, but used only $7,185/year of mortgage interest and property tax and left out New York's SUA. That gave $7,305/year of net income instead of $194.38/month, so the benefit was $115/month instead of $239.80."
+us,scenario_118,snap,grok-build-0.1,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the $780.67/month of SSI income and treated shelter costs as unlisted, even though mortgage interest and property tax were given. It also used 2024 parameters ($193 standard deduction, $291 maximum). Correct net income is $194.38, giving $239.80/month."
+us,scenario_118,snap,inkling,prompt_ambiguity,taxable_income_or_deductions,False,"Counted only $233 of Social Security and left out the $780.67/month of SSI, so the shelter deduction zeroed net income and it paid the $298 maximum. With SSI counted, net income is $194.38 and the benefit is $239.80/month."
+us,scenario_118,snap,kimi-k2.6,prompt_ambiguity,taxable_income_or_deductions,False,"Treated the $2,800 of Social Security as the only income and left out the $780.67/month of SSI, and used an inflated $309 maximum. With SSI counted, net income is $194.38 and the benefit is $298 − $58.20 = $239.80/month."
+us,scenario_118,snap,kimi-k3,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated SSI at $10,412/year (federal SSI is $9,368) and applied no shelter deduction, wrongly claiming housing costs were not given. Mortgage interest, property tax and the NY SUA give a $610.62 uncapped deduction and $194.38 of net income, so the benefit is $239.80/month."
+us,scenario_118,snap,minimax-m3,prompt_ambiguity,taxable_income_or_deductions,False,"Contradicted itself by calling the income too low to qualify and then returning a $0 benefit. Under SNAP, lower net income produces a larger allotment. The household's $194.38 of net income gives $298 − $58.20 = $239.80/month."
+us,scenario_118,snap,muse-spark-1.3,prompt_ambiguity,taxable_income_or_deductions,False,"Overstated gross income at about $1,101 (correct SSI plus Social Security is $1,014) and applied no excess shelter deduction, even though mortgage interest, property tax and the NY SUA are uncapped shelter costs. The correct $610.62 deduction gives $194.38 of net income and $239.80/month."
+us,scenario_118,snap,muse-wonder,prompt_ambiguity,taxable_income_or_deductions,False,"Counted SSI and the $209 standard deduction correctly, but its $196 excess shelter deduction used only mortgage interest and property tax and left out New York's SUA. With the SUA, the deduction is $610.62 and net income is $194.38, so the benefit is $239.80/month, not $115.44."
+us,scenario_118,snap,ox-alpha,prompt_ambiguity,taxable_income_or_deductions,False,"Got $1,014 of income, the $209 deduction and $805 of adjusted income right, but set shelter costs at $598.75 without New York's SUA, so excess shelter was $196.25 instead of $610.62. Net income is $194.38, not $608.75, so the benefit is $239.80/month."
+us,scenario_118,snap,qwen-3.7-max,prompt_ambiguity,taxable_income_or_deductions,False,"Left out the $780.67/month of SSI income and never applied the uncapped excess shelter deduction for the listed mortgage interest and property tax. It also used a $198 standard deduction and a $291 maximum. Correct net income is $194.38, giving $239.80/month."
+us,scenario_118,snap,qwen3.8-max,prompt_ambiguity,taxable_income_or_deductions,False,"Wrongly treated a qualifying child as a requirement for SNAP. A one-person elderly/disabled household is eligible, here categorically through SSI, and its $194.38 of net income gives $239.80/month."
us,scenario_118,ssi,claude-fable-5,llm_error,other,False,"Its reasoning reaches the correct figure, 12 × ($994 − $213.33) = $9,368, but then switches to an incoherent mix of a $725.33 benefit rate and an expression built on $967. It submitted $6,144, which no valid SSI calculation supports."
us,scenario_118,ssi,claude-haiku-4.5,llm_error,period_annualization,False,"It treated $2,800 of annual Social Security as exceeding a monthly benefit rate of about $943 and concluded the person is ineligible. Monthly, that income is only $233.33, or $213.33 after the $20 exclusion, well below the 2026 rate of $994, so SSI of $780.67/month is payable."
us,scenario_118,ssi,claude-haiku-5.5,llm_error,period_annualization,False,"It said $2,800 of annual Social Security exceeds the SSI income threshold. Converted to monthly, countable income is $213.33 ($233.33 − $20), far below the $994 benefit rate, which leaves a $780.67/month federal payment ($9,368/year)."
@@ -10190,25 +10301,26 @@ us,scenario_118,state_refundable_credits,muse-wonder,llm_error,state_local_rule,
us,scenario_118,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model checked only the NY EITC and the child and dependent care credit. It missed the refundable real property tax credit, which pays this senior homeowner the $375 maximum."
us,scenario_118,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model took zero NY taxable income and no earnings or children to mean no refundable credits. The real property tax credit applies to this 74-year-old homeowner and pays the $375 age-65+ maximum.
us,scenario_118,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model cited the household's age yet found no credit, missing that age 65+ plus $1,634 of property taxes qualifies for the refundable NY real property tax credit. That credit pays the $375 elderly maximum."
-us,scenario_119,child1_chip_eligible,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The model put income at about 197% FPL by subtracting the traditional 401(k) and IRA contributions to reach a MAGI near $52,569. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model placed income at about 197% FPL, using a MAGI reduced by the 401(k) and IRA contributions, and called that safely under the 200% + 5% disregard limit. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model used a MAGI of $52,569, which is $55,800 minus the $3,087 traditional 401(k) and $144 IRA contributions, and got about 197% FPL. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model said household income was below Virginia's 205% FPL FAMIS threshold without showing a calculation. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model asserted income was within the FAMIS threshold without computing the household's FPL ratio. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model assumed MAGI was under the 205% FPL FAMIS limit. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,The model gave no parseable value for child1_chip_eligible. The correct answer is 0 because Child 1's employer-sponsored insurance disqualifies the child from CHIP.
-us,scenario_119,child1_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model was right that income is above Virginia's child Medicaid limit and within its CHIP income range: the engine counts $52,569 in household MAGI. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model relied on an estimated CHIP income range without comparing actual countable income to the limit. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,gpt-6.1-terra,llm_error,thresholds_rates,False,"The model relied on the 5% income disregard to keep the household under the FAMIS limit. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model computed MAGI at about 192% FPL, using income reduced by the retirement contributions. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model correctly saw income above Virginia's child Medicaid limit but assumed it was below the 200–205% FAMIS limit. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model estimated MAGI at 190–197% FPL by subtracting the pre-tax retirement contributions. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model cut MAGI to $46,180 (168% FPL) by subtracting the $6,389 ESI premiums as well as the 401(k) and IRA contributions. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model gave no parseable value for child1_chip_eligible. The correct answer is 0 because Child 1's employer-sponsored insurance disqualifies the child from CHIP.
-us,scenario_119,child1_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used a 3-person MAGI of $52,569, net of the 401(k) and IRA contributions, and placed it within the CHIP band. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"The model estimated income at about 191% FPL, inside what it called a 148–200% FAMIS band. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
-us,scenario_119,child1_chip_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used a MAGI of $52,569, net of the 401(k) and IRA contributions, and got about 204% FPL, just under the 205% FAMIS limit. The engine counts $52,569 in household MAGI, within Virginia's CHIP income limit. Child 1's employer-sponsored insurance disqualifies the child from CHIP."
+us,scenario_119,child1_chip_eligible,claude-fable-5.1,llm_error,health_coverage,False,"Checked only age (under 19) and income (~197% FPL against the 200% FAMIS limit). It ignored that Child 1 has employer-sponsored insurance. Existing group coverage bars the child from being a CHIP targeted low-income child, so the correct answer is ineligible."
+us,scenario_119,child1_chip_eligible,claude-fable-5.2,llm_error,health_coverage,False,"Explicitly stated that existing employer coverage is not a bar, which is the exact error. A child covered by an employer group health plan fails CHIP's uninsured requirement. Under PolicyEngine's CHIP test that makes Child 1 ineligible even though the ~192% FPL income is under the FAMIS limit."
+us,scenario_119,child1_chip_eligible,claude-opus-5.5,llm_error,health_coverage,False,Based eligibility only on income (~197% FPL against 200% plus the 5% disregard). It never applied the CHIP rule that a child with employer-sponsored coverage is not a targeted low-income child. Child 1's ESI makes them CHIP-ineligible.
+us,scenario_119,child1_chip_eligible,claude-sonnet-5.5,llm_error,health_coverage,False,"Correctly found Child 1 Medicaid-ineligible and MAGI of $52,569 below 200% FPL. It then skipped CHIP's categorical requirement that the child be uninsured. Child 1 has employer-sponsored insurance, which disqualifies them from CHIP."
+us,scenario_119,child1_chip_eligible,gemini-3.5-flash,llm_error,health_coverage,False,"Checked only age under 19 and income below 205% FPL. It ignored the listed employer-sponsored insurance, which excludes Child 1 from CHIP under the rule that CHIP covers only uninsured children."
+us,scenario_119,child1_chip_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"Treated age 14 and income within the FAMIS threshold as enough for eligibility. It missed that Child 1's employer-sponsored coverage fails CHIP's no-other-coverage requirement, so the correct result is ineligible."
+us,scenario_119,child1_chip_eligible,gemini-3.7-flash,llm_error,health_coverage,False,"Applied only the age test and the 205% FPL income test. It never accounted for Child 1 having employer-sponsored insurance, and that group coverage disqualifies the child from CHIP."
+us,scenario_119,child1_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,"Returned no usable value or explanation for child1_chip_eligible. The correct derivation is 0: Child 1 is above Virginia's child Medicaid limit, but the child's employer-sponsored insurance bars CHIP eligibility."
+us,scenario_119,child1_chip_eligible,gpt-5.5,llm_error,health_coverage,False,"Placed Child 1 in the income band between the Medicaid and CHIP limits and stopped there. It ignored the child's employer-sponsored insurance, which fails CHIP's uninsured requirement and makes the child ineligible."
+us,scenario_119,child1_chip_eligible,gpt-5.6-luna,llm_error,health_coverage,False,"Concluded eligibility from age under 19 and income within the Virginia CHIP range alone. It missed the categorical bar: Child 1 is covered by employer-sponsored insurance, so they do not qualify for CHIP."
+us,scenario_119,child1_chip_eligible,gpt-6-luna,llm_error,health_coverage,False,"Tested only age and MAGI against Virginia's upper CHIP limit. It never applied the rule that children with existing group health coverage are excluded, and Child 1's employer-sponsored insurance makes them CHIP-ineligible."
+us,scenario_119,child1_chip_eligible,gpt-6.1-terra,llm_error,health_coverage,False,"Relied on the age limit and the income test with the 5% disregard. It ignored that Child 1 has employer-sponsored insurance, which fails CHIP's uninsured-child requirement and produces ineligibility."
+us,scenario_119,child1_chip_eligible,grok-4.5,llm_error,health_coverage,False,"Found MAGI at ~192% FPL under the 200% FAMIS limit and declared Child 1 eligible. It disregarded the child's employer-sponsored coverage, and that coverage excludes the child from CHIP as a targeted low-income child."
+us,scenario_119,child1_chip_eligible,grok-4.6,llm_error,health_coverage,False,"Placed income between Virginia's child Medicaid and FAMIS limits but never checked insurance status. Child 1's employer-sponsored insurance fails CHIP's no-other-coverage condition, so the child is ineligible."
+us,scenario_119,child1_chip_eligible,grok-4.7,llm_error,health_coverage,False,"Applied the age test and an income test of 190–197% FPL against the 200–205% FAMIS limit. It missed that Child 1 is covered by employer-sponsored insurance, which disqualifies them from CHIP."
+us,scenario_119,child1_chip_eligible,grok-build-0.1,llm_error,health_coverage,False,"Understated MAGI at $46,180 (168% FPL) and then used only the income band to declare eligibility. It ignored Child 1's employer-sponsored insurance, which bars CHIP enrollment regardless of income."
+us,scenario_119,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no usable value or explanation for child1_chip_eligible. The correct derivation is 0: Child 1 is above the child Medicaid limit, and their employer-sponsored insurance disqualifies them from CHIP."
+us,scenario_119,child1_chip_eligible,kimi-k3,llm_error,health_coverage,False,"Correctly computed MAGI of $52,569 inside Virginia's CHIP band but treated the income test as decisive. It ignored that Child 1 has employer-sponsored insurance, which fails CHIP's uninsured requirement and makes them ineligible."
+us,scenario_119,child1_chip_eligible,muse-spark-1.3,llm_error,health_coverage,False,"Placed income at ~191% FPL inside the 148–200% FAMIS band and stopped. It never applied CHIP's exclusion of children with employer group coverage, which disqualifies Child 1."
+us,scenario_119,child1_chip_eligible,ox-alpha,llm_error,health_coverage,False,"Put MAGI at ~204% FPL, just under a 205% limit, and declared eligibility on income alone. It ignored the listed employer-sponsored insurance for Child 1, and that coverage bars the child from CHIP."
us,scenario_119,child1_early_head_start_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_119,child1_head_start_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_119,child1_medicaid_eligible,claude-fable-5,llm_error,health_coverage,False,"The model computed MAGI correctly ($55,800 less $3,231 of traditional 401(k)/IRA contributions ≈ 1.9x FPL) and correctly rejected Virginia's children's Medicaid MAGI limit of 143% + 5% disregard for ages 6-18, then reversed its own conclusion by asserting that PolicyEngine counts coverage up to ~205% FPL as Medicaid. Virginia's coverage above 143% FPL is FAMIS, a separate CHIP program that PolicyEngine scores under child1_chip_eligible and never under is_medicaid_eligible; the CHIP-funded Medicaid expansion for 6-18 year olds stops at the same 143%/148% line the model had already ruled out, so medicaid_category is NONE at 1.92x FPL."
@@ -10219,25 +10331,26 @@ us,scenario_119,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missin
us,scenario_119,child1_medicare_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_119,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_119,child1_wic_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_119,child2_chip_eligible,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The model put income at about 197% FPL by subtracting the 401(k) and IRA contributions to reach a MAGI of $52,569. The engine also counts a MAGI of $52,569, about 192% of the 2026 3-person FPL of $27,320, so Child 2 passes Virginia's 205% FAMIS income test. Child 2 fails CHIP because they have employer-sponsored insurance, which the engine treats as disqualifying coverage, and the model never tested coverage."
-us,scenario_119,child2_chip_eligible,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model said income was about 197% FPL, which only works if the traditional 401(k) and IRA contributions are taken out of MAGI. The engine also takes those contributions out, for a MAGI of about $52,569, which is about 192% of the 2026 3-person FPL and within Virginia's 205% FAMIS ceiling. Child 2 fails CHIP because their employer-sponsored insurance is disqualifying coverage, which the model never considered."
-us,scenario_119,child2_chip_eligible,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model used a MAGI of $52,569 (wages plus interest minus $3,087 of 401(k) and $144 of IRA contributions) and got 197% FPL, under 200%. The engine counts the same MAGI, about 192% of the 2026 3-person FPL of $27,320, which is within Virginia's 205% CHIP limit. The model correctly found Child 2 not Medicaid-eligible but missed that Child 2's employer-sponsored insurance disqualifies them from CHIP."
-us,scenario_119,child2_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model asserted that income was under Virginia's 205% FAMIS threshold without computing a ratio. The engine's MAGI of about $52,569 is about 192% of the 2026 3-person FPL, so Child 2 passes the income test. Child 2 fails CHIP because their employer-sponsored insurance is disqualifying coverage, which the model never considered."
-us,scenario_119,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model said income was 'within the CHIP threshold' with no income-to-FPL calculation. Child 2 meets the age test and the income test, because the engine's MAGI of about $52,569 is about 192% of the 2026 3-person FPL and within Virginia's 205% FAMIS limit. Child 2 is not CHIP-eligible because their employer-sponsored insurance is disqualifying coverage."
-us,scenario_119,child2_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model assumed MAGI was within FAMIS's 205% FPL limit but never measured it. The engine's MAGI of about $52,569 for a 3-person household is about 192% FPL, within the limit. Child 2 fails CHIP because their employer-sponsored insurance is disqualifying coverage, which the model never considered."
-us,scenario_119,child2_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no usable value or explanation for child2_chip_eligible. The correct answer is 0, because this 11-year-old has employer-sponsored insurance, which disqualifies them from CHIP, and the child is also not Medicaid-eligible."
-us,scenario_119,child2_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model claimed income was within Virginia's CHIP/FAMIS limit without computing the income-to-FPL ratio. The engine's MAGI of about $52,569 is about 192% of the 2026 3-person FPL, within that limit. Child 2 is ineligible because their employer-sponsored insurance is disqualifying coverage, which the model never considered."
-us,scenario_119,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model said income was within an 'estimated' Virginia CHIP range without testing it against the actual ceiling. The engine's MAGI of about $52,569 is about 192% of the 2026 3-person FPL, within the 205% FAMIS limit. Child 2 fails CHIP because their employer-sponsored insurance is disqualifying coverage, which the model never considered."
-us,scenario_119,child2_chip_eligible,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model's estimate of MAGI fell below Virginia's CHIP upper limit, which matches reducing income for the 401(k) and IRA contributions. The engine also reduces income for those contributions, to about $52,569, which is about 192% of the 2026 3-person FPL and within the 205% FAMIS limit. Child 2 is not CHIP-eligible because their employer-sponsored insurance is disqualifying coverage."
-us,scenario_119,child2_chip_eligible,gpt-6.1-terra,llm_error,thresholds_rates,False,"The model relied on the 5-percentage-point income disregard to pull income under the CHIP ceiling. The engine's MAGI of about $52,569 is about 192% of the 2026 3-person FPL, within Virginia's 205% FAMIS limit. Child 2 is ineligible because their employer-sponsored insurance is disqualifying coverage, which the model never considered."
-us,scenario_119,child2_chip_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model put MAGI at about 192% FPL by subtracting the 401(k) and IRA contributions and dividing by a higher projected 2026 guideline. The engine also reaches about 192% FPL and applies Virginia's 205% FAMIS limit, so Child 2 passes the income test. Child 2 fails CHIP because their employer-sponsored insurance is disqualifying coverage, which the model never considered."
-us,scenario_119,child2_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model was right that income is above Virginia's child Medicaid limit and below the FAMIS ceiling. The engine's MAGI of about $52,569 is about 192% of the 2026 3-person FPL, within Virginia's 205% FAMIS limit. Child 2's employer-sponsored insurance is disqualifying coverage for CHIP, so Child 2 qualifies for neither program."
-us,scenario_119,child2_chip_eligible,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model's MAGI estimate of 190-197% FPL comes from subtracting the retirement contributions from income. The engine also subtracts those contributions, for a MAGI of about $52,569, which is about 192% of the 2026 3-person FPL and within Virginia's 205% FAMIS limit. Child 2 fails CHIP because their employer-sponsored insurance is disqualifying coverage, which the model never considered."
-us,scenario_119,child2_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model got a MAGI of $46,180 (168% FPL) by subtracting the $6,389 ESI premium as well as the 401(k) and IRA contributions from $55,800. The engine subtracts only the 401(k) deferral and IRA deduction, for a MAGI of about $52,569, which is about 192% of the 2026 3-person FPL and within Virginia's 205% CHIP limit. Child 2 fails CHIP because their employer-sponsored insurance is disqualifying coverage."
-us,scenario_119,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no usable value or explanation for child2_chip_eligible. The correct answer is 0, because this 11-year-old has employer-sponsored insurance, which disqualifies them from CHIP, and the child is also not Medicaid-eligible."
-us,scenario_119,child2_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used a 3-person MAGI of $52,569, which nets the traditional 401(k) and IRA contributions out of wages plus interest, and placed it inside Virginia's CHIP band. The engine counts the same $52,569, about 192% of the 2026 3-person FPL and within the 205% FAMIS limit. Child 2 is ineligible because their employer-sponsored insurance is disqualifying coverage, which the model never considered."
-us,scenario_119,child2_chip_eligible,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"The model put income at about 191% FPL by reducing MAGI for retirement contributions and using a higher guideline, then placed it in the 148-200% FAMIS band. The engine also reduces MAGI for those contributions, to about $52,569, which is about 192% of the 2026 3-person FPL and within Virginia's 205% FAMIS ceiling. Child 2 fails CHIP because their employer-sponsored insurance is disqualifying coverage, which the model never considered."
-us,scenario_119,child2_chip_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used a MAGI of $52,569 and got about 204% FPL, just under the 205% limit. The engine counts the same $52,569 MAGI, which is about 192% of the 2026 3-person FPL of $27,320 and within Virginia's 205% FAMIS limit. Child 2 is not CHIP-eligible because their employer-sponsored insurance is disqualifying coverage, which the model never considered."
+us,scenario_119,child2_chip_eligible,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The model put income at about 197% FPL by netting the 401(k) and IRA contributions out of MAGI, then compared that to a 200% FAMIS limit. PolicyEngine counts household income at about $55,800, which is above Virginia's CHIP child income limit, so Child 2 fails the income test."
+us,scenario_119,child2_chip_eligible,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"The model used an income of about 192% FPL, built from a deduction-reduced MAGI, and placed it under a 205% FAMIS ceiling. PolicyEngine measures countable household income at about $55,800, which is over Virginia's CHIP child limit, so the child is ineligible even though the age test passes."
+us,scenario_119,child2_chip_eligible,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The model's 197% FPL figure comes from a MAGI with the retirement contributions subtracted, and it treated that as within the FAMIS limit. PolicyEngine's CHIP income test uses household income of about $55,800, which is above Virginia's child CHIP threshold, so the answer is not eligible."
+us,scenario_119,child2_chip_eligible,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"The model computed MAGI as $52,569 by subtracting the traditional 401(k) and IRA contributions, which gave 197% FPL against a 200% limit. PolicyEngine counts household income at about $55,800, which is above Virginia's CHIP child limit, so the model was right that the child is not Medicaid-eligible but wrong to then call the child CHIP-eligible."
+us,scenario_119,child2_chip_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model asserted that household income was under Virginia's 205% FAMIS threshold without testing the actual income. PolicyEngine's countable income of about $55,800 for a 3-person household is above Virginia's CHIP child limit, so Child 2 fails the income test."
+us,scenario_119,child2_chip_eligible,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model checked only age and assumed income was within the FAMIS threshold without computing it. PolicyEngine counts household income at about $55,800, which is above Virginia's CHIP child income limit, so the child is ineligible."
+us,scenario_119,child2_chip_eligible,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model claimed MAGI fell under Virginia's 205% FAMIS limit but never applied the income test properly. PolicyEngine measures household income at about $55,800 for 3 people, which is above Virginia's CHIP child limit, so the answer is not eligible."
+us,scenario_119,child2_chip_eligible,glm-5.3,parse_contract_failure,missing_output,False,"The model returned no parseable value for child2_chip_eligible. The correct value is 0, because household income of about $55,800 is above Virginia's CHIP child income limit and the child is not Medicaid-eligible."
+us,scenario_119,child2_chip_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model said household income was within the FAMIS child limit without stating an income figure. PolicyEngine's countable household income of about $55,800 is above Virginia's CHIP child threshold for a 3-person household, so Child 2 fails the income test."
+us,scenario_119,child2_chip_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model relied on an 'estimated' Virginia CHIP range and placed the household inside it without checking. PolicyEngine counts income at about $55,800, which is above Virginia's CHIP child limit, so the child is ineligible."
+us,scenario_119,child2_chip_eligible,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"The model's estimated MAGI was below Virginia's CHIP upper limit only because it understated countable income. PolicyEngine puts household income at about $55,800, which is above Virginia's CHIP child limit, so Child 2 is not eligible."
+us,scenario_119,child2_chip_eligible,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"The model added the 5% income disregard to a MAGI that had already been reduced by the retirement contributions, which brought the child under the limit. PolicyEngine counts household income at about $55,800, which is above Virginia's CHIP child limit, so the child fails the income test."
+us,scenario_119,child2_chip_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model put MAGI at about 192% FPL, using a deduction-reduced income and a higher poverty guideline, and compared it to a 200% FAMIS limit. PolicyEngine counts household income at about $55,800, which is above Virginia's CHIP child limit, so the answer is not eligible."
+us,scenario_119,child2_chip_eligible,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model placed MAGI between Virginia's child Medicaid limit and the 200–205% FAMIS limit. PolicyEngine counts household income at about $55,800, which is above the CHIP ceiling, so the model got Medicaid ineligibility right but should not have concluded CHIP eligibility."
+us,scenario_119,child2_chip_eligible,grok-4.7,llm_error,taxable_income_or_deductions,False,"The model's 190–197% FPL estimate comes from a MAGI with the 401(k) and IRA contributions netted out, which placed it under the FAMIS limit. PolicyEngine counts household income at about $55,800, which is above Virginia's CHIP child limit, so Child 2 is ineligible."
+us,scenario_119,child2_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model got a MAGI of $46,180 by subtracting the $6,389 employer health premiums as well as the retirement contributions from income, which yielded 168% FPL. PolicyEngine counts household income at about $55,800, which is above Virginia's CHIP child limit, so the child fails the income test."
+us,scenario_119,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no parseable value for child2_chip_eligible. The correct value is 0, because household income of about $55,800 is above Virginia's CHIP child income limit and the child is not Medicaid-eligible."
+us,scenario_119,child2_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used a MAGI of $52,569 (wages plus interest, minus the traditional 401(k) and IRA contributions) and placed it inside Virginia's CHIP band. PolicyEngine counts household income at about $55,800, which is above Virginia's CHIP child limit, so the child is not eligible."
+us,scenario_119,child2_chip_eligible,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"The model's 191% FPL figure comes from a deduction-reduced MAGI and fits inside its assumed 148–200% FAMIS band. PolicyEngine counts household income at about $55,800, which is above Virginia's CHIP child limit, so Child 2 fails the income test."
+us,scenario_119,child2_chip_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used a MAGI of $52,569, put it at 204% FPL, and judged it just under a 205% limit. PolicyEngine counts household income at about $55,800, which is above Virginia's CHIP child limit, so the marginal pass flips to not eligible."
us,scenario_119,child2_early_head_start_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_119,child2_head_start_eligible,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_119,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model substituted Virginia's FAMIS/CHIP ceiling (200%, effectively 205% FPL with the 5% disregard) for the children's MAGI Medicaid limit, which Virginia sets at 143% FPL for ages 6-18. Its own arithmetic put household income above the 200% figure it cited, and it then overrode that with a ""marginal overage"" fudge and a CHIP-to-Medicaid conflation to return 1. Child2's MAGI of 1.92 x FPL exceeds 143% FPL, so no MAGI child category applies and medicaid_category is NONE."
@@ -10423,57 +10536,58 @@ us,scenario_119,state_refundable_credits,kimi-k3,llm_error,state_local_rule,Fals
us,scenario_119,state_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"Estimated 20% of about $1,490 federal EITC as refundable. Virginia does not stack the refundable and nonrefundable EITC, and this household's positive Virginia liability makes the nonrefundable credit the elected one. So state refundable credits are $0."
us,scenario_119,state_refundable_credits,muse-wonder,llm_error,state_local_rule,False,"Reported about 20% of a roughly $1,096 federal EITC as a refundable Virginia credit. It ignored the election between Virginia's nonrefundable and refundable EITC. With Virginia tax absorbing the nonrefundable credit, the household takes the nonrefundable credit and refundable credits are $0."
us,scenario_119,tanf,glm-5.3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"It reached about $200,500 of taxable income but reported $30,238. That is roughly $10,000 below what the 2026 single brackets give on that income (about $40,800), so its bracket arithmetic was badly wrong. It also capped SALT at the old $10,000 instead of using the $40,400 cap, took the full $3,744 of mortgage interest instead of $2,808 under the $750,000 limit, and left out the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,750 refund in AGI and used no IRA deduction, giving AGI of $234,414 and taxable income of $198,819. The frozen trace's IRA deduction was the adjudicated engine defect. Its NIIT estimate used a $1,853 base. This output remains excluded from scoring."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It took a made-up $30,000 standard deduction and used head-of-household rates for a single filer. It then subtracted $11,297 of estimated withholding, which is a tax payment. The frozen trace reports $39,858 of regular tax plus $163.40 NIIT; this output remains excluded from scoring."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rounded $25,000 guess without applying the brackets. The frozen trace used $196,918 of taxable income, $39,858 of regular tax and $163.40 of NIIT. This output remains excluded, so its difference from the frozen value is ungraded."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It computed $40,823, then cut it to $38,740 with no credit to justify the cut. Its itemized deductions capped SALT at the old $10,000 instead of using the $40,400 cap for 2026. It also took the full $3,744 of mortgage interest instead of $2,808 under the $750,000 debt limit, skipped the 0.5%-of-AGI charitable floor, and left out the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It used the old $10,000 SALT cap, ignored the 0.5% charitable floor and started the 24% bracket at $107,400. The 2026 threshold is $105,700. Its taxable income was $202,030. The frozen trace used $13,908 of SALT and $163.40 of NIIT; state income tax paid and NIIT scope remain unresolved, so this output is excluded."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It used only $2,012 of property tax for SALT, deducted the full $3,744 of mortgage interest and skipped the charitable floor, producing $208,465 of taxable income. The frozen trace used $13,908 of SALT and $2,808 of mortgage interest. State income tax paid and NIIT scope remain unresolved, so this output is excluded."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,750 refund in AGI and used $13,679 of SALT and $198,906 of taxable income. The frozen trace used $13,908 of SALT and $196,918 of taxable income. Its NIIT estimate used a $1,853 base. This output remains excluded because NIIT inclusion and state income tax paid are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It assumed TCJA expired and applied pre-TCJA brackets, a personal exemption, uncapped SALT and miscellaneous deductions above a 2% floor. OBBBA made the TCJA structure permanent, so 2026 has no exemption and no miscellaneous deductions. It also left Connecticut income tax out of SALT, and its submitted $42,046 matches none of its own intermediate results."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"It used AGI of $232,635, including the IRA deduction identified as a defect in the adjudication. It used only the stated property tax for SALT and skipped the charitable floor. Its $24,500 is far below the roughly $42,900 the 2026 single brackets give on its own $207,651 of taxable income. This output remains excluded because state income tax paid and NIIT inclusion are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It omitted the $617 traditional 401(k) deferral, included the $1,750 refund and used no IRA deduction, giving AGI of $235,031 and taxable income of $199,649. The frozen trace's IRA deduction was the adjudicated engine defect. Its $70 NIIT estimate differs from $163.40 in the frozen trace; this output remains excluded from scoring."
-us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $7,429 of employer health insurance premiums from wages, which the reference keeps in taxable wages, and skipped the 0.5% charitable floor, getting taxable income of $192,198. Its $34,322 is also about $4,400 below the 2026 bracket tax on that taxable income ($38,725), and it left out the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset law, with a personal exemption, pre-TCJA brackets and a deduction for unreimbursed employee expenses above a 2% floor. OBBBA made the TCJA structure permanent for 2026. It also subtracted $7,429 of employer health premiums from wages and figured NIIT on $1,853 instead of $4,300."
-us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It applied pre-TCJA rules: a $5,250 personal exemption and the restored 15%/25%/28%/33% brackets. OBBBA kept the 10/12/22/24% brackets and no exemption for 2026. Its itemized deductions were $42,059, compared with $35,717 in the frozen trace, and its NIIT used a $1,853 base. This output remains excluded because NIIT inclusion and state income tax paid are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,750 refund in AGI and used no IRA deduction, giving AGI of $234,414. It deducted the full $20,164 of charity without the 2026 0.5%-of-AGI floor. Its omission of the IRA deduction agrees with the adjudication, which identified the frozen trace's deduction as an engine defect. This output remains excluded because NIIT inclusion and state income tax paid are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It assumed TCJA expired in 2026, so it applied no SALT cap, a $2,966 miscellaneous deduction for employee expenses and pre-TCJA brackets. Under OBBBA, the TCJA brackets stay and miscellaneous itemized deductions are permanently disallowed. It also ignored the $750,000 mortgage limit ($2,808 allowed), skipped the 0.5% charitable floor and left out NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation. The frozen trace used AGI of $232,635, itemized deductions of $35,717, taxable income of $196,918, regular tax of $39,858 and NIIT of $163.40. This output is excluded, so its difference from the frozen value is ungraded."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $7,429 of employer health premiums from wages and claimed miscellaneous deductions for unreimbursed employee expenses, which OBBBA permanently disallows. Its NIIT of $87 covers only the $2,300 of interest and leaves the $2,000 of rental income out of the $4,300 NIIT base. Its regular tax also overshoots the 2026 bracket tax on its own reduced taxable income."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It applied TCJA-sunset rules: a $5,000 personal exemption, unreimbursed employee expenses deducted above a 2% floor, and pre-TCJA brackets. OBBBA kept the TCJA brackets with no exemption. It also subtracted $7,429 of employer health premiums from wages and figured NIIT on $1,853 instead of $4,300."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation. The frozen trace reports $39,858 of regular tax on $196,918 plus $163.40 NIIT. This output is excluded, so its difference from the frozen value is ungraded."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $7,429 of employer health premiums from wages, giving AGI of $226,985. It used the old $10,000 SALT cap and skipped the charitable floor. The frozen trace used AGI of $232,635, SALT of $13,908 and NIIT of $163.40. This output remains excluded because NIIT inclusion and state income tax paid are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation. Its $37,824 is about $2,198 below the frozen $40,021.82. This output is excluded, so that difference is ungraded."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation. Its $33,023.76 is about $7,000 below the frozen $40,021.82. This output is excluded, so that difference is ungraded."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-4-argon,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a personal exemption and applied pre-TCJA rates for 2026. OBBBA permanently eliminated the exemption and kept the 10/12/22/24% brackets. The frozen trace reports $39,858 of tax on $196,918 plus $163.40 NIIT; this output remains excluded from scoring."
-us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It used only $2,012 of property tax for SALT and skipped the charitable floor, producing itemized deductions of $24,984 and taxable income of $209,430. Its AGI included the $1,750 refund. The frozen trace used $13,908 of SALT and excluded the refund. This output remains excluded because state income tax paid and NIIT inclusion are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It counted mortgage interest twice ($7,488) and skipped the $750,000 debt limit; the frozen trace used $2,808. It used the old $10,000 SALT cap and skipped the charitable floor. The frozen trace's IRA deduction was an adjudicated defect, and NIIT scope and state income tax paid remain unresolved; this output is excluded."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"It said the itemized deductions were limited by the standard deduction. The frozen trace selected $35,717 of itemized deductions because they exceeded the $18,150 standard deduction. Its $38,141 is about $1,881 below the frozen value; this output is excluded and that difference is ungraded."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It gave only a vague description. Its $35,233 is about $4,789 below the frozen $40,021.82. This output is excluded, so that difference is ungraded."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied post-sunset rules, with a $5,300 personal exemption and pre-TCJA rates. Under OBBBA, 2026 keeps the TCJA brackets and has no exemption. Its itemized deductions were $40,615, compared with $35,717 in the frozen trace, and its NIIT used about a $1,853 base. This output remains excluded because NIIT inclusion and state income tax paid are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It used the 2026 brackets on $210,286 of taxable income. The frozen trace used $196,918 of taxable income, $13,908 of SALT and $163.40 of NIIT. State income tax paid and NIIT scope remain unresolved, so this output is excluded."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It used AGI of about $234,414 and included about $70 of NIIT. Its implied taxable income of about $195,800 differs from $196,918 in the frozen trace. This output remains excluded because NIIT inclusion and state income tax paid are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It brought back the personal exemption and used pre-TCJA rates on the assumption that TCJA expired in 2026. OBBBA made the TCJA brackets permanent with no exemption, so $196,918 of taxable income yields $39,858 plus $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $7,429 of employer health premiums from wages, giving AGI of $226,985 instead of $232,635; the reference keeps those premiums in taxable wages. It also left out the $163.40 NIIT. Its $38,627.54 equals the 2026 bracket tax on about $191,790 of taxable income."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It used $40,221 of itemized deductions and included the $1,750 refund in AGI. The frozen trace used $35,717 of itemized deductions and excluded the refund. This output remains excluded because NIIT inclusion and state income tax paid are unresolved. Its $39,204 is exactly the 2026 bracket tax on $194,193."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"Its $39,672.32 is the 2026 bracket tax on $196,143 of taxable income. The model included the $1,750 refund in AGI. The frozen trace left it out, used $196,918 of taxable income and added $163.40 of NIIT. This output remains excluded because NIIT inclusion and state income tax paid are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $7,429 of employer health premiums from wages, giving AGI of $226,985 instead of $232,635, and left out the $163.40 NIIT. Its $38,437.74 equals the 2026 bracket tax on $190,999 instead of $196,918."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6.1-terra,reference_engine_defect,taxable_income_or_deductions,False,"It submitted $40,182 without a breakdown of AGI, taxable income or NIIT. That is $160.18 above the frozen, ungraded reference. Its final amount does not establish its refund or NIIT treatment."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It showed no computation. Its $39,250 is about $772 below the frozen $40,021.82. This output is excluded, so that difference is ungraded."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied post-sunset rules: a $5,427 personal exemption, $2,966 of miscellaneous employee-expense deductions, full mortgage interest and pre-TCJA brackets. OBBBA kept the TCJA brackets, permanently disallowed miscellaneous deductions and the exemption, and kept the $750,000 mortgage limit."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It assumed TCJA expired, so it brought back a $5,437 personal exemption and the 15%/25%/28% brackets. OBBBA made the 10/12/22/24% brackets and the zero exemption permanent. Its $38,146 of itemized deductions exceeds $35,717 in the frozen trace; this output is excluded and the frozen amount is ungraded."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It used $15,371 of SALT and the full $20,164 of charity without the 0.5% floor, producing $194,321 of taxable income. The frozen trace used $13,908 of SALT and $19,001 of charity. Its NIIT used a $1,853 base. This output remains excluded because NIIT inclusion and state income tax paid are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It applied post-TCJA-expiration brackets and a $2,966 miscellaneous deduction for employee expenses above a 2% floor. OBBBA kept the TCJA brackets and permanently disallowed those deductions. It also took the full $3,744 of mortgage interest instead of $2,808 and left out NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It claimed a $2,966 miscellaneous deduction for unreimbursed employee expenses, which OBBBA permanently disallows. It also skipped the 0.5% charitable floor and used $14,212 of SALT, producing $194,235 of taxable income. Its $163 NIIT matches the rounded NIIT amount in the frozen trace, and it included the $1,750 refund in AGI. This output remains excluded because NIIT inclusion and state income tax paid are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was nothing to score against $40,021.82."
-us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value or explanation for federal_income_tax_before_refundable_credits, so there was nothing to score against $40,021.82."
-us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It reported $0 with no reasoning. The frozen trace used AGI of $232,635 and tax of $40,021.82. This output is excluded, so the frozen comparison is ungraded."
-us,scenario_120,federal_income_tax_before_refundable_credits,muse-spark-1.3,reference_engine_defect,taxable_income_or_deductions,False,"It took the full $3,744 of mortgage interest instead of $2,808 under the $750,000 debt limit. It also deducted the full $20,164 of charity without the 0.5% floor, put the $1,750 refund into AGI and left out the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,muse-wonder,reference_engine_defect,taxable_income_or_deductions,False,"It itemized about $23,800 and used about $211,000 of taxable income. The frozen trace used $35,717 of itemized deductions, $196,918 of taxable income and $163.40 of NIIT. This output remains excluded because state income tax paid and NIIT inclusion are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It included the $1,750 refund, used no IRA deduction and deducted $3,744 of mortgage interest, producing $197,685 of taxable income. Its IRA treatment agrees with the adjudication, which identified the frozen trace's deduction as an engine defect. Its $40,042 is $20.18 above the frozen reference; this output remains excluded because NIIT inclusion and state income tax paid are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It used an $18,525 standard deduction, did not subtract the $617 401(k) deferral and used the 2025 bracket threshold of $103,350. The frozen trace itemized $35,717 and included $163.40 of NIIT. This output remains excluded because NIIT inclusion and state income tax paid are unresolved."
-us,scenario_120,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It treated a single filer as married filing separately and counted the $6,147 tax-exempt pension as taxable. It also deducted the full $20,164 of charity on top of the standard deduction, exceeding the $1,000 limit for non-itemizers, and subtracted $750 of retirement credit even though its income eliminates the elderly or disabled credit. Its $48,557 does not even match its own $45,287 minus $750."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,reference_engine_defect,taxable_income_or_deductions,False,"Its own taxable income of about $200,500 comes to roughly $40,700 under the 2026 single brackets (24% bracket from $105,700), but it reported $30,238, a bracket-arithmetic error of about $10,500. It also capped SALT at $10,000 instead of the $40,400 OBBBA cap, deducted the full $3,744 of mortgage interest instead of the $2,808 allowed by the $750k limit, skipped the 0.5% charitable floor, and left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5.1,reference_engine_defect,taxable_income_or_deductions,False,"It put the $1,750 state tax refund into AGI ($234,414), but the reference AGI is $232,635 without it, so its taxable income came out at $198,819 instead of $196,918. It also netted the farm rent loss into the NIIT base and charged NIIT on $1,853 ($70) instead of on $4,300 of interest and rental income ($163.40)."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It left the $2,447 farm rent loss out of AGI and taxed the $1,750 state refund, which pushed AGI up to $236,832 (reference: $232,635). It also lowered SALT to $9,761 by guessing at withholding, when the actual Connecticut income tax plus property tax is $13,908, so taxable income came out at $205,283 instead of $196,918."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $11,297 of estimated withholding, which is a payment and not a nonrefundable credit, so it has no place in tax before refundable credits. It also took a made-up $30,000 standard deduction instead of the $35,717 in itemized deductions, counted 100% of Social Security instead of the taxable $35,615, and applied head-of-household rates to a single filer."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It gave a rough $25,000 guess without running the brackets. The correct path is $196,918 of taxable income after $35,717 of itemized deductions, which gives $39,858 of regular tax plus $163.40 NIIT. Its number understates the liability by about $15,000, which matches never applying the 24% bracket to income above $105,700."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It capped SALT at the old $10,000 instead of using the $40,400 OBBBA cap ($13,908 deductible), deducted all $3,744 of mortgage interest despite the $750k limit ($2,808), skipped the 0.5% charitable floor, and taxed the $1,750 state refund. It then cut its own computed $40,823 down to $38,740 for no stated reason and left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,reference_engine_defect,taxable_income_or_deductions,False,"It capped SALT at $10,000 instead of using the $40,400 OBBBA cap that allows the full $13,908, deducted the whole $20,164 of charitable gifts without the 0.5%-of-AGI floor, and put the $1,750 state refund into AGI. As a result, taxable income came out at $202,030 instead of $196,918, and it also left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,reference_engine_defect,taxable_income_or_deductions,False,"It left the roughly $11,900 Connecticut income tax out of SALT and deducted only the $2,012 property tax. It also took all $3,744 of mortgage interest instead of the $2,808 allowed by the $750k limit and skipped the 0.5% charitable floor. Its $208,465 of taxable income actually comes to about $43,165 under 2026 brackets, not the $40,650 it reported, and it left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It put the $1,750 state tax refund into AGI, which the reference leaves out, so taxable income came out at $198,906 instead of $196,918. It also netted the farm rent loss into the NIIT base and charged $70 instead of 3.8% of $4,300 ($163.40)."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It assumed the TCJA expired after 2025 and used pre-TCJA 10/15/25/28/33% brackets, a personal exemption, no SALT cap, and 2%-floor miscellaneous deductions. The OBBBA made the TCJA rates, the zero personal exemption, and the suspension of miscellaneous deductions permanent. Its final $42,046 also contradicts its own computed $47,383."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $232,635 matches the reference, but it left Connecticut income tax out of SALT and skipped the 0.5% charitable floor. Then, instead of applying the brackets to its own $207,651 of taxable income (which gives about $42,900), it reported a guessed $24,500 and left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,reference_engine_defect,taxable_income_or_deductions,False,"Its AGI of $235,031 did not subtract the $617 traditional 401(k) deferral and included the $1,750 state refund, so taxable income came out at $199,649 instead of $196,918. It also netted the farm loss into NIIT and charged $70 instead of $163.40 on $4,300 of interest and rent."
+us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted $7,429 of employer-sponsored insurance premiums from the gross wages given in the facts, which PolicyEngine does not do, and it taxed the $1,750 state refund. Its own taxable income of $192,198 comes to about $38,725 under 2026 brackets, not the $34,322 it reported, and it left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,reference_engine_defect,taxable_income_or_deductions,False,"It applied rules that assume the TCJA expired: pre-TCJA brackets, a $5,063 personal exemption, and $3,114 of 2%-floor miscellaneous deductions. Under the OBBBA, the TCJA brackets stay in place for 2026, the personal exemption is $0, and miscellaneous deductions remain suspended. It also wrongly subtracted $7,429 of employer health premiums from the given gross wages."
+us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,reference_engine_defect,taxable_income_or_deductions,False,"It claimed a $5,250 personal exemption and used brackets that assume the TCJA expired, but the OBBBA made the TCJA rate schedule and the zero personal exemption permanent. It also inflated itemized deductions to $42,059 (reference: $35,717) and netted the farm loss into NIIT ($70 instead of $163.40)."
+us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the $1,750 state refund (AGI $234,414 instead of $232,635) and deducted the full $20,164 of charitable gifts without the OBBBA 0.5%-of-AGI floor ($19,001 allowed), so taxable income came out at $198,557 instead of $196,918. It also left out the $163.40 NIIT on $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly assumed the TCJA expired, so it removed the SALT cap, restored $2,966 of 2%-floor miscellaneous employee-expense deductions, and used pre-TCJA rates. The OBBBA keeps the TCJA brackets permanently, keeps miscellaneous deductions suspended, and limits mortgage interest to the $750k share ($2,808 rather than $3,744)."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $39,591 is $431 below the reference, which matches leaving out the $163.40 NIIT and deducting the full $20,164 of charitable gifts without the $1,163 OBBBA 0.5%-of-AGI floor."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of employer health premiums from the given gross wages (AGI $226,956 instead of $232,635) and deducted miscellaneous employee business expenses, which the OBBBA permanently eliminated. It also charged NIIT of $87 instead of $163.40 on $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,reference_engine_defect,taxable_income_or_deductions,False,"It used rules that assume the TCJA expired: a $5,000 personal exemption and $3,149 of 2%-floor unreimbursed employee expenses. The OBBBA keeps the TCJA brackets, the zero exemption, and the suspension of miscellaneous deductions in place. It also subtracted the $7,429 of employer premiums from gross wages and charged NIIT on the net $1,853 instead of $4,300."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $33,814 is about $6,200 below the reference, which is the same as understating taxable income by about $25,000 at the 24% marginal rate relative to the correct $196,918 (AGI $232,635 minus $35,717 itemized) and leaving out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of employer health premiums from the given gross wages and capped SALT at the old $10,000, when the $40,400 OBBBA cap allows the full $13,908. It also skipped the 0.5% charitable floor and left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.7-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $37,824 equals 2026 bracket tax on about $188,400 of taxable income, roughly $8,500 below the correct $196,918. That gap matches subtracting the $7,429 of employer health premiums from the given gross wages and leaving out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.8-flash,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $33,023.76 equals 2026 bracket tax on about $168,400 of taxable income, which is roughly $28,500 below the correct $196,918 (AGI $232,635 minus $35,717 itemized), and it adds no NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-4-argon,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted a personal exemption, so it applied rules that assume the TCJA expired. The OBBBA made the $0 personal exemption and the TCJA 10/12/22/24/32% brackets permanent, and the correct taxable income of $196,918 gives $39,858 plus $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,reference_engine_defect,taxable_income_or_deductions,False,"It left the roughly $11,900 Connecticut income tax out of SALT and deducted only the $2,012 property tax. It also skipped the 0.5% charitable floor and taxed the $1,750 refund, so taxable income came out at $209,430 instead of $196,918 and part of it landed in the 32% bracket. It also left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.3,reference_engine_defect,taxable_income_or_deductions,False,"It double-counted mortgage interest at $7,488 by adding the head's $3,744 to the tax-unit $2,976 and $768, and it ignored the $750k debt limit that allows only $2,808. It capped SALT at $10,000 instead of the full $13,908 allowed under the $40,400 OBBBA cap, skipped the 0.5% charitable floor, and left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,reference_engine_defect,taxable_income_or_deductions,False,"Its reasoning wrongly treats itemized deductions as limited by the standard deduction, when itemized deductions of $35,717 simply replace the $18,150 standard deduction. Its $38,141 equals bracket tax on about $189,800 of taxable income, roughly $7,200 below the correct $196,918, and it adds no $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,reference_engine_defect,taxable_income_or_deductions,False,"It cites retirement-savings and other deductions or credits that do not apply. The senior deduction is fully phased out at this MAGI, and no nonrefundable credit is available. Its $35,233 matches taxable income about $19,000 below the correct $196,918 and leaves out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly used rules that assume the TCJA expired, with a $5,300 personal exemption and pre-TCJA rates. The OBBBA made the TCJA brackets and the $0 personal exemption permanent for 2026. It also inflated itemized deductions to $40,615 (reference: $35,717) and charged NIIT of only $70 instead of $163.40."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,reference_engine_defect,taxable_income_or_deductions,False,"Its taxable income of $210,286 implies about $24,100 of itemized deductions, which means it left the roughly $11,900 Connecticut income tax out of SALT (the deductible SALT total is $13,908 under the $40,400 cap). That error pushed income into the 32% bracket, and it also left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the $1,750 state refund in a $234,414 AGI, yet its regular tax implies about $38,600 of itemized deductions instead of the correct $35,717. It also netted the farm loss into the NIIT base and charged about $70 instead of 3.8% of $4,300 ($163.40)."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,reference_engine_defect,taxable_income_or_deductions,False,"It explicitly restored the personal exemption and applied 2026 rules that assume the TCJA expired. The OBBBA permanently kept the TCJA 10/12/22/24/32% schedule and the $0 exemption, and under that schedule $196,918 of taxable income gives $39,858 plus $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-astra,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of employer health premiums from the given gross wages, which lowered AGI to $226,985 instead of $232,635. Its tax implies about $191,800 of taxable income instead of $196,918, and it left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-luna,reference_engine_defect,taxable_income_or_deductions,False,"It overstated itemized deductions at $40,221 against the correct $35,717 ($13,908 SALT + $2,808 mortgage interest + $19,001 charity after the 0.5% floor) and taxed the $1,750 state refund. It also left out the $163.40 NIIT on $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6-sol,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the $1,750 state refund (AGI $234,414) and then took about $38,300 of itemized deductions instead of $35,717, which gave $196,143 of taxable income. It also left out the $163.40 NIIT on $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6.1-sol,reference_engine_defect,taxable_income_or_deductions,False,"It subtracted the $7,429 of employer health premiums from the given gross wages (AGI $226,985 instead of $232,635), so taxable income came out at about $191,000 instead of $196,918. It also left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-6.1-terra,reference_engine_defect,taxable_income_or_deductions,False,"Its $40,182 is $160 above the reference, which matches taxing the $1,750 state tax refund that PolicyEngine leaves out of AGI while leaving out the $163.40 NIIT on $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,reference_engine_defect,taxable_income_or_deductions,False,"It gave no derivation. Its $39,250 equals 2026 bracket tax on about $194,400 of taxable income, which means deductions overstated by about $2,500 compared with the correct $35,717 itemized total, and it adds none of the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,reference_engine_defect,taxable_income_or_deductions,False,"It applied rules that assume the TCJA expired: a $5,427 personal exemption, $2,966 of 2%-floor miscellaneous deductions, full $3,744 mortgage interest, and pre-TCJA brackets. The OBBBA keeps the TCJA rates, the $0 exemption, the suspension of miscellaneous deductions, and the $750k mortgage limit permanently."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.6,reference_engine_defect,taxable_income_or_deductions,False,"It restored a $5,437 personal exemption and taxed income at the pre-TCJA 10/15/25/28% brackets on the assumption that the TCJA expired. The OBBBA made the TCJA 10/12/22/24/32% schedule and the $0 exemption permanent."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.7,reference_engine_defect,taxable_income_or_deductions,False,"It overstated SALT at $15,371 (correct: $13,908) and deducted the full $20,164 of charitable gifts without the 0.5%-of-AGI floor, which lowered taxable income to $194,321 instead of $196,918. It also netted the farm loss into NIIT ($70 instead of $163.40)."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,reference_engine_defect,taxable_income_or_deductions,False,"It applied brackets that assume the TCJA expired and deducted $2,966 of 2%-floor miscellaneous employee expenses plus all $3,744 of mortgage interest. The OBBBA permanently keeps the TCJA rates, keeps miscellaneous deductions suspended, and limits mortgage interest to the $750k share ($2,808)."
+us,scenario_120,federal_income_tax_before_refundable_credits,inkling,reference_engine_defect,taxable_income_or_deductions,False,"It deducted $2,966 of unreimbursed employee business expenses, which the OBBBA permanently eliminated, and it took the full $20,164 of charitable gifts without the 0.5%-of-AGI floor. It also taxed the $1,750 refund, so taxable income came out at $194,235 instead of $196,918. Its $163.40 NIIT is right."
+us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value for federal_income_tax_before_refundable_credits, so no answer could be scored against the $40,021.82 reference."
+us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value for federal_income_tax_before_refundable_credits, so no answer could be scored against the $40,021.82 reference."
+us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,reference_engine_defect,taxable_income_or_deductions,False,"It reported $0 tax for a single filer with $232,635 of AGI. The correct path itemizes $35,717 of deductions, leaving $196,918 of taxable income, which gives $39,858 of regular tax plus $163.40 NIIT, and no nonrefundable credits offset it."
+us,scenario_120,federal_income_tax_before_refundable_credits,muse-spark-1.3,reference_engine_defect,taxable_income_or_deductions,False,"It deducted the full $3,744 of mortgage interest despite the $1M of debt exceeding the $750k limit ($2,808 allowed), took charity without the 0.5% floor, and overstated SALT. Itemized deductions came to about $38,920 instead of $35,717, and it left out the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,muse-wonder,reference_engine_defect,taxable_income_or_deductions,False,"Its itemized deductions of about $23,800 count only property tax under SALT and leave out the roughly $11,900 Connecticut income tax, which is fully deductible under the $40,400 OBBBA cap. That inflated taxable income to about $211,000 instead of $196,918."
+us,scenario_120,federal_income_tax_before_refundable_credits,ox-alpha,reference_engine_defect,taxable_income_or_deductions,False,"It taxed the $1,750 state refund and deducted all $3,744 of mortgage interest instead of the $2,808 allowed by the $750k limit, which gave $197,685 of taxable income. It also left out the $163.40 NIIT on $4,300 of interest and rental income."
+us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,reference_engine_defect,taxable_income_or_deductions,False,"It took an $18,525 standard deduction instead of itemizing $35,717 (SALT $13,908, mortgage interest $2,808, charity $19,001). It also failed to subtract the $617 401(k) deferral and used 2025 bracket thresholds instead of 2026."
+us,scenario_120,federal_income_tax_before_refundable_credits,qwen3.8-max,reference_engine_defect,taxable_income_or_deductions,False,"It treated a lone filer as married filing separately and taxed the $6,147 of tax-exempt pension. It also took the standard deduction plus charity instead of itemizing and claimed a retirement income credit that does not apply. Its $48,557 also contradicts its own $45,287 − $750 computation."
us,scenario_120,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -10483,100 +10597,102 @@ us,scenario_120,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_ou
us,scenario_120,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model returned 0 without applying Medicare's age-based eligibility rule. At age 76, the head exceeds the age-65 threshold, so the eligibility value is 1 regardless of the household's income, assets, or employer-sponsored coverage."
us,scenario_120,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_120,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got every part right: SS $10,267.01, Medicare $2,401.16, CT Paid Leave 0.5% about $828, for a total of $13,496.16. It then submitted $13,736.63 anyway. That number has no basis in its own calculation, so this is an inconsistency between the final answer and the reasoning."
-us,scenario_120,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It counted only federal SS ($10,267.01) and Medicare ($2,401.16) and left out Connecticut's mandatory 0.5% employee Paid Leave contribution ($827.98). That contribution is part of employee payroll tax."
-us,scenario_120,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It made up a 0.64% Connecticut rate and then swapped it for an invented '-$48.79 net from itemizations' adjustment, which pushed the total below plain FICA. The correct state piece is Connecticut's 0.5% Paid Leave contribution on all $165,597 of wages ($827.98), added on top of $12,668.17 in FICA."
-us,scenario_120,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"It rounded the 7.65% FICA total ($12,668.17) down to roughly $12,000 and never added Connecticut's mandatory 0.5% employee Paid Leave contribution ($827.98)."
-us,scenario_120,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It said Connecticut has no extra mandatory employee payroll tax. That leaves out the CT Paid Leave contribution of 0.5% on covered wages ($827.98), so its total stopped at the FICA amount of $12,668.17."
-us,scenario_120,payroll_tax,claude-opus-5,llm_error,other,False,"It correctly found FICA of $12,668 and the CT Paid Leave contribution of $828, which add up to $13,496. It then submitted $10,680, a number that does not follow from those parts and comes from an unexplained 'PolicyEngine adjustment'."
-us,scenario_120,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,It left out the Connecticut Paid Leave contribution because it assumed PolicyEngine does not model it. The CT 0.5% employee paid leave contribution ($827.98) is a mandatory employee state payroll tax and belongs in the total.
-us,scenario_120,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It wrongly stated that Connecticut has no mandatory employee-side payroll tax. CT requires a 0.5% employee Paid Leave contribution ($827.98 here), which belongs on top of the $12,668.17 in FICA."
-us,scenario_120,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It deliberately left out all state payroll tax and so missed Connecticut's mandatory 0.5% employee Paid Leave contribution ($827.98) on $165,597 of wages."
-us,scenario_120,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It cut FICA wages to $157,551 by subtracting the $7,429 employer insurance premium and the $617 traditional 401k deferral, but neither reduces the payroll tax base here. FICA applies to all $165,597. It also wrongly said Connecticut has no employee payroll tax, which leaves out the $827.98 Paid Leave contribution."
-us,scenario_120,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It used $158,168 as the taxable wage by subtracting the $7,429 employer-sponsored insurance premium. In this computation, SS, Medicare and CT Paid Leave all apply to the full $165,597, so every component came out too low."
-us,scenario_120,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It reported only SS and Medicare ($12,668.17) and left out Connecticut's mandatory 0.5% employee Paid Leave contribution ($827.98)."
-us,scenario_120,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its $14,197 is about $700 above the correct total. That is consistent with adding an Additional Medicare Tax-style amount even though wages of $165,597 are below the $200,000 threshold. The correct total is 7.65% FICA ($12,668.17) plus the 0.5% CT Paid Leave contribution ($827.98)."
-us,scenario_120,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It correctly included CT's 0.5% Paid Leave contribution but applied every rate to $158,168, after subtracting the $7,429 employer insurance premium. The payroll tax base is the full $165,597 of wages."
-us,scenario_120,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It treated the $7,429 employer-sponsored insurance premium as an exclusion from FICA wages and used a $158,168 base. All three components apply to the full $165,597, which gives $13,496.16."
-us,scenario_120,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"It said the total was just SS and Medicare, yet its $14,352.19 works out to about 8.67% of wages instead of 7.65%. Its arithmetic is wrong, and it never identified the 0.5% CT Paid Leave contribution that makes up the rest of the correct $13,496.16."
-us,scenario_120,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It applied 6.2%, 1.45% and 0.5% to $158,168 after taking out the $7,429 employer insurance premium. Those rates apply to the full $165,597, so each component was too low."
-us,scenario_120,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,891 equals 8.15% of $158,168, which shows it took the $7,429 employer insurance premium out of covered wages. SS, Medicare and CT Paid Leave all apply to the full $165,597."
-us,scenario_120,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"Its $12,891.10 matches applying 6.2% + 1.45% + 0.5% to $158,168, so it subtracted the $7,429 employer insurance premium from wages. The correct base for all three is $165,597."
-us,scenario_120,payroll_tax,gemini-4-argon,llm_error,state_local_rule,False,"It counted only SS and Medicare ($12,668.17) and left out Connecticut's mandatory 0.5% employee Paid Leave contribution ($827.98)."
-us,scenario_120,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"It subtracted the employer insurance premium, the traditional 401k deferral and even the Roth 401k contribution from FICA wages. 401k deferrals are subject to FICA, and here the payroll base is the full $165,597. It also left out the $827.98 CT Paid Leave contribution."
-us,scenario_120,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It wrongly stated that Connecticut has no employee payroll tax and so left out the mandatory 0.5% Paid Leave contribution ($827.98) on top of $12,668 in FICA."
-us,scenario_120,payroll_tax,gpt-5.4-mini,llm_error,other,False,"Its $10,087 is less than the Social Security tax alone (6.2% × $165,597 = $10,267.01), so the arithmetic is wrong. It also wrongly said Connecticut has no employee payroll tax, leaving out the $827.98 Paid Leave contribution."
-us,scenario_120,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It named the right 6.2% and 1.45% rates but reported $6,180, about half of the $12,668.17 those rates produce on $165,597. It also left out the $827.98 CT Paid Leave contribution."
-us,scenario_120,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"Its SS and Medicare figures are correct, but it set the CT Paid Leave contribution at $922.50, which is 0.5% of the $184,500 wage cap. The contribution applies to actual covered wages: 0.5% × $165,597 = $827.98."
-us,scenario_120,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"It subtracted the $617 traditional 401(k) deferral from FICA wages, but elective deferrals remain subject to SS and Medicare. It also left out Connecticut's 0.5% employee Paid Leave contribution ($827.98)."
-us,scenario_120,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It used $158,168 as payroll-taxable wages by subtracting the $7,429 employer insurance premium. SS, Medicare and CT Paid Leave apply to the full $165,597, which gives $13,496.16."
-us,scenario_120,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the $7,429 employer health premium as a pretax exclusion and taxed only $158,168. The payroll tax base in this computation is the full $165,597 of wages for all three components."
-us,scenario_120,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It applied only federal SS and Medicare rates ($12,668) and left out Connecticut's mandatory 0.5% employee Paid Leave contribution ($827.98)."
-us,scenario_120,payroll_tax,grok-4.5,llm_error,state_local_rule,False,It explicitly said there is no employee state payroll tax in CT. That leaves out the mandatory 0.5% CT Paid Leave contribution ($827.98) that belongs on top of 7.65% FICA.
-us,scenario_120,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,It said there are 'no other employee payroll taxes' and so left out Connecticut's 0.5% employee Paid Leave contribution ($827.98).
-us,scenario_120,payroll_tax,inkling,llm_error,state_local_rule,False,It wrongly stated that CT has no mandatory employee state payroll tax. The CT Paid Leave contribution of 0.5% of wages ($827.98) is required and belongs in the total.
-us,scenario_120,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for payroll_tax, so there is no answer to score."
-us,scenario_120,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value or explanation for payroll_tax, so there is no answer to score."
-us,scenario_120,payroll_tax,minimax-m3,llm_error,other,False,"It reported $0 even though the head earns $165,597 in wages, which are subject to 6.2% SS, 1.45% Medicare and the 0.5% CT Paid Leave contribution, for a total of $13,496.16."
-us,scenario_120,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It stated that CT has no mandatory employee payroll tax and so left out the 0.5% CT Paid Leave contribution ($827.98) on top of $12,668 in FICA."
-us,scenario_120,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It wrongly stated that Connecticut has no mandatory employee state payroll tax, leaving out the 0.5% Paid Leave contribution ($827.98) from the total."
-us,scenario_120,payroll_tax,qwen3.8-max,llm_error,other,False,"It reported only a Social Security figure, wrongly capped at $9,655, although wages are below the wage base and the correct SS amount is $10,267.01. It also left out Medicare ($2,401.16) and the CT Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning correctly added SS $10,267.01, Medicare $2,401.16 and CT PFML 0.5% ($827.99) to get $13,496.16. It then submitted $13,736.63, which is $240.47 higher and matches nothing in its own work, so the final value contradicts its own calculation."
+us,scenario_120,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed only federal FICA ($10,267.01 + $2,401.16 = $12,668.17). It left out Connecticut's mandatory 0.5% Paid Leave employee contribution on $165,597 ($827.98), which is part of employee state payroll tax."
+us,scenario_120,payroll_tax,claude-fable-5.2,llm_error,state_local_rule,False,"It said outright that no state employee payroll tax applies and stopped at FICA of $12,668.17. That misses Connecticut's mandatory 0.5% Paid Leave contribution of $827.98."
+us,scenario_120,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"It used an outdated $168,600 wage base and misdid the arithmetic: SS came out at $10,267.14 and Medicare at $2,401.06. It then invented a negative CT payroll adjustment of -$48.79, when CT actually imposes a positive 0.5% Paid Leave contribution of $827.98 on full wages."
+us,scenario_120,payroll_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"It rounded FICA to roughly $12,000 instead of computing 7.65% × $165,597 = $12,668.17 exactly. It also left out Connecticut's 0.5% Paid Leave employee contribution ($827.98)."
+us,scenario_120,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"It concluded CT has no mandatory employee payroll tax and submitted the FICA-only $12,668.17. Connecticut's Paid Leave program requires a 0.5% employee contribution on the full $165,597 ($827.98)."
+us,scenario_120,payroll_tax,claude-opus-5,llm_error,other,False,"It correctly found FICA of $12,668 and the CT paid-leave contribution of $828, which sum to $13,496. It then replaced that with an unexplained $10,680, an arbitrary adjustment that matches no payroll tax rule."
+us,scenario_120,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,It deliberately left out the CT paid-leave contribution on the false assumption that PolicyEngine doesn't model it. PolicyEngine counts the 0.5% Connecticut employee contribution ($827.98) as employee state payroll tax.
+us,scenario_120,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It claimed Connecticut has no mandatory employee-side payroll tax and stopped at $12,668.17. That ignores the CT Paid Leave 0.5% employee contribution ($827.98 on $165,597)."
+us,scenario_120,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It explicitly counted no state payroll tax and submitted FICA only ($12,668.17). It missed Connecticut's mandatory 0.5% Paid Leave contribution of $827.98."
+us,scenario_120,payroll_tax,deepseek-v4-flash-0731,llm_error,payroll_tax_base,False,"It cut the FICA base to $157,551 by subtracting both the ESI premium and the traditional 401(k) deferral. 401(k) deferrals stay subject to FICA, and PolicyEngine taxes the full $165,597. It also wrongly said CT has no employee payroll tax and omitted the 0.5% Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"It included CT PFML but reduced FICA and PFML wages to $158,168 by subtracting the $7,429 ESI premium. PolicyEngine applies SS, Medicare and the CT 0.5% contribution to the full $165,597 gross wages."
+us,scenario_120,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It summed only SS ($10,267.01) and Medicare ($2,401.16) to get $12,668.17. It omitted Connecticut's mandatory 0.5% Paid Leave employee contribution of $827.98."
+us,scenario_120,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its reasoning brings in a 0.9% Additional Medicare Tax 'adjusted for this individual', but wages of $165,597 are below the $200,000 threshold, so none applies. Its $14,197 overshoots the correct SS + Medicare + 0.5% CT Paid Leave total of $13,496.16."
+us,scenario_120,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"It correctly included CT PFML at 0.5% but applied all rates to $158,168 after subtracting the $7,429 ESI premium. PolicyEngine's payroll tax base is the full $165,597 of gross wages."
+us,scenario_120,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"It subtracted the $7,429 ESI premium to get FICA wages of $158,168, which understates every component. SS, Medicare and CT Paid Leave apply to the full $165,597, giving $13,496.16."
+us,scenario_120,payroll_tax,gemini-3.5-flash-lite,llm_error,other,False,"It claimed to include only SS and Medicare, which come to 7.65% × $165,597 = $12,668.17. It reported $14,352.19 instead, an arithmetic error, and it never added the CT 0.5% Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"It computed SS, Medicare and CT PFML on $158,168 after taking the ESI premium out of FICA wages. Applied to the full $165,597, the same rates give $13,496.16."
+us,scenario_120,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,891 is exactly 8.15% of $158,168, so it removed the $7,429 ESI premium from the payroll base. SS, Medicare and CT Paid Leave all apply to the full $165,597 of wages."
+us,scenario_120,payroll_tax,gemini-3.8-flash,llm_error,payroll_tax_base,False,"Its $12,891.10 equals 8.15% of $158,168, so it shrank the payroll base by the $7,429 ESI premium. All three components apply to the full $165,597 gross wages, totaling $13,496.16."
+us,scenario_120,payroll_tax,gemini-4-argon,llm_error,state_local_rule,False,"It summed only federal SS and Medicare ($12,668.17). It left out Connecticut's mandatory 0.5% Paid Leave employee contribution ($827.98)."
+us,scenario_120,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"It subtracted the ESI premium, the traditional 401(k) deferral and even the Roth 401(k) contribution from FICA wages. Both kinds of 401(k) contribution stay FICA-taxable, and PolicyEngine taxes the full $165,597. It also omitted the CT 0.5% Paid Leave contribution."
+us,scenario_120,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It wrongly said CT has no employee payroll tax and submitted FICA only ($12,668). That omits the mandatory 0.5% Connecticut Paid Leave contribution of $827.98."
+us,scenario_120,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"It wrongly said Connecticut has no mandatory employee payroll tax, missing the 0.5% Paid Leave contribution ($827.98). Its $10,087 is also below even the FICA-only amount of 7.65% × $165,597 = $12,668.17."
+us,scenario_120,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It named the right 6.2% and 1.45% rates on wages under the base, but its $6,180 is less than half of 7.65% × $165,597 = $12,668.17, an arithmetic error. It also omitted the CT 0.5% Paid Leave contribution."
+us,scenario_120,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"It computed the CT paid-leave contribution as $922.50, which is 0.5% of the $184,500 wage cap. The 0.5% applies to actual covered wages of $165,597, giving $827.98."
+us,scenario_120,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"It subtracted the $617 traditional 401(k) deferral from FICA wages, but elective deferrals stay subject to SS and Medicare tax. It also omitted Connecticut's 0.5% Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,gpt-6-astra,llm_error,payroll_tax_base,False,"It applied 8.15% (SS + Medicare + CT paid leave) to $158,168 after subtracting the ESI premium. PolicyEngine applies these rates to the full $165,597 gross wages."
+us,scenario_120,payroll_tax,gpt-6.1-sol,llm_error,payroll_tax_base,False,"It treated the $7,429 ESI premium as a pretax exclusion from payroll wages and used $158,168. The payroll tax base is the full $165,597, which gives SS $10,267.01, Medicare $2,401.16 and CT Paid Leave $827.98."
+us,scenario_120,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It computed only federal FICA at 7.65% ($12,668). It omitted Connecticut's mandatory 0.5% Paid Leave employee contribution ($827.98)."
+us,scenario_120,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It explicitly said CT has no employee state payroll tax and submitted 7.65% × $165,597. That misses the 0.5% Connecticut Paid Leave contribution of $827.98."
+us,scenario_120,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,It concluded there are no other employee payroll taxes beyond SS and Medicare. Connecticut's mandatory 0.5% Paid Leave contribution adds $827.98.
+us,scenario_120,payroll_tax,inkling,llm_error,state_local_rule,False,"It wrongly said CT has no mandatory employee state payroll tax and stopped at $12,668. It omitted the 0.5% Connecticut Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for payroll_tax, so there was no answer to score."
+us,scenario_120,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value or explanation for payroll_tax, so there was no answer to score."
+us,scenario_120,payroll_tax,minimax-m3,llm_error,other,False,"It gave $0 even though the head has $165,597 of wages, which are subject to 6.2% SS ($10,267.01), 1.45% Medicare ($2,401.16) and the 0.5% CT Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It wrongly said CT has no mandatory employee payroll tax and submitted FICA only ($12,668). That omits Connecticut's 0.5% Paid Leave contribution of $827.98."
+us,scenario_120,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It explicitly said Connecticut has no mandatory employee state payroll tax and stopped at FICA of $12,668.17. It missed the 0.5% CT Paid Leave contribution ($827.98)."
+us,scenario_120,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It treated employee SS tax as capped at $9,655, which implies a wage base below the head's wages and does not match the 2026 $184,500 base. It also left out the 1.45% Medicare tax ($2,401.16) and the CT 0.5% Paid Leave contribution ($827.98) entirely."
us,scenario_120,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,snap,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It never worked out the CT Social Security subtraction (taxable SS minus 25% of benefits = $25,140) and added back $2,852 of tax-exempt interest, which is not a CT addition here. It then gave an unanchored rate estimate. The correct base is $207,524, which yields $11,239 of graduated tax plus $680 of add-backs; its $12,130 is $211 too high."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,thresholds_rates,False,"It applied the $25,140 Social Security adjustment correctly but did not subtract the $1,750 state tax refund, leaving CT AGI at $209,274 instead of $207,524. It also added only $430 ($250 phase-out + $180 recapture) and left out the separate $250 low-tax recapture, so its add-backs total $430 instead of $680."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"It invented a CT deduction for federal income tax paid and an over-65 deduction, and also applied federal itemized deductions. Connecticut has none of these and taxes the $207,524 of CT AGI directly once the personal exemption phases out. Its $0 ignores $11,239 of graduated tax plus $680 of add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,other,False,"It gave no derivation. The correct figure is $11,239 of graduated tax on $207,524 of CT taxable income plus $680 of phase-out and recapture add-backs. Its $7,000 fits an assumption of large exemptions or credits, all of which are fully phased out at this income."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It denied any Social Security subtraction, although CT caps taxable benefits at 25% for single filers over $75,000 (a $25,140 subtraction). It also added back $2,852 of tax-exempt interest and skipped the $1,750 refund subtraction, inflating CT AGI to $236,883. It then used a 3% first bracket and cut its $13,247 subtotal to $11,890 with no stated basis, instead of taxing $207,524 and adding $680."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It used the pre-2024 3% and 5% rates for the first two brackets. It also took an unexplained ~$10,000 off its ~$210k CT AGI, though no CT deduction remains once the personal exemption is phased out. Its $9,970 is below even the plain graduated tax on its own $200k base and leaves out the $680 of add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It said CT rates run only from 2% to 5.5%, leaving out the 6% bracket ($100k–$200k) and the 6.5% bracket above $200k. It also put CT taxable income at ~$198,800 instead of $207,524 (federal AGI minus the $25,140 SS adjustment and $1,750 refund). Its $9,450 is below the $11,239 graduated tax and omits the $680 of add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"It got CT AGI ($207,495) and graduated tax ($11,237) essentially right but added only $250 for the 2% bracket phase-out plus $180 of recapture. It left out the separate $250 low-tax recapture, so add-backs total $430 instead of $680 and the answer is $252 short."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It concluded that no Social Security subtraction applies above $100,000. In fact CT caps taxable benefits at 25% for higher-income single filers, which gives a $25,140 subtraction. It also skipped the $1,750 refund subtraction, so it taxed $234,414 instead of $207,524, and it left out the $250 phase-out add-back and $430 of recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It never settled the Social Security subtraction and claimed the personal exemption and personal credit lower liability, though both are fully phased out at this income. Even on its own ~$198,000 base the CT schedule gives about $10,600, so $7,200 reflects misapplied brackets. The correct result is $11,239 on $207,524 plus $680 of add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,thresholds_rates,False,"It overstated CT AGI at $209,891 because it did not fully apply the $1,750 state refund subtraction alongside the $25,140 SS adjustment; the correct CT AGI is $207,524. It added only $180 of recapture and left out the $250 2% bracket phase-out add-back and the $250 low-tax recapture, so add-backs total $180 instead of $680."
-us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security, although above $75,000 CT subtracts only taxable SS minus 25% of benefits ($25,140). It also added back $2,852 of tax-exempt interest, applied a $15,000 personal exemption that is fully phased out at this income, and left out the $680 of phase-out and recapture add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It exempted Social Security entirely instead of applying the $25,140 adjustment, which leaves 25% of benefits taxable. It then subtracted a $1,549 personal tax credit that is fully phased out at this CT AGI, and it left out the $680 of 2% bracket phase-out and recapture add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It subtracted the state refund but left out the $25,140 Social Security adjustment, so it taxed $232,664 instead of $207,524. That pushes $25,140 of extra income into the 6.5% bracket and overstates tax by about $1,254."
-us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security instead of $25,140. Above the $75,000 single threshold, CT keeps 25% of benefits ($10,475) taxable, so CT AGI is $207,524, not $197,049. It also used the pre-2024 3%/5% rates and left out the $680 of add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It inverted the CT Social Security rule and subtracted only 25% of taxable SS ($8,904). The correct subtraction is the amount by which taxable SS exceeds 25% of benefits ($35,615 − $10,475 = $25,140), so CT AGI was overstated at $223,731 instead of $207,524."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"It gave no derivation. The correct computation taxes $207,524 of CT taxable income at 2%/4.5%/5.5%/6%/6.5% ($11,239) and adds $680 of phase-out and recapture. Its $7,800 fits an assumption of large exemptions or a much smaller base, and every such relief is fully phased out at this income."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It ruled out any Social Security subtraction because of high income. CT still subtracts $25,140 for single filers above $75,000, because taxable benefits are capped at 25% of benefits. It also skipped the $1,750 refund subtraction and taxed roughly $234k instead of $207,524."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It understated federal AGI at $225,235 (the correct figure is $234,414) and subtracted 75% of taxable SS ($26,711) instead of taxable SS minus 25% of benefits ($25,140), giving CT AGI of $198,524 instead of $207,524. Its $10,961 base tax also uses the pre-2024 3%/5% rates, and its phase-out and recapture add-backs total only $150 of the required $680."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"It gave no derivation. Graduated tax on the correct $207,524 base is $11,239, and the full $680 of add-backs ($250 phase-out + $250 low-tax + $180 middle-tax recapture) brings it to $11,919. Its $11,770 is $149 short, which fits counting only part of those add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its CT taxable income of $212,320 is $4,796 above the correct $207,524 because it understated the CT subtractions; the $25,140 SS adjustment plus the $1,750 refund total $26,890. Its implied add-backs of $660 are also $20 short of $680."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It gave no derivation. Its $13,125 is $1,206 above the correct figure, which matches taxing about $18,500 too much income at the 6.5% marginal rate. That fits skipping most of the $26,890 of CT subtractions (the $25,140 SS adjustment plus the $1,750 refund)."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,other,False,"It gave no derivation. Its $9,706.75 is more than $2,200 below the correct figure, which fits subtracting all Social Security or applying a phased-out exemption and dropping the $680 of add-backs. The correct figure is $11,239 of graduated tax on $207,524 plus $680."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It treated the CT rule as a 75% exemption of gross benefits ($31,425). The correct subtraction is taxable SS minus 25% of benefits ($35,615 − $10,475 = $25,140). This understated CT AGI by about $6,285, and its total came out $399 short."
-us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"It fully exempted Social Security instead of applying the $25,140 adjustment and subtracted a $15,000 'standard deduction', which Connecticut does not have. It also took a $200 property tax credit that is fully phased out at this AGI and left out the $680 of phase-out and recapture add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,glm-5.3,llm_error,thresholds_rates,False,"It subtracted all $35,615 of taxable SS instead of $25,140, then applied a flat 4.5% rate to $197,049. The correct method uses CT's graduated 2%/4.5%/5.5%/6%/6.5% brackets on $207,524 and adds $680 of phase-out and recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It gave no derivation. Its $4,414 is less than 40% of the correct graduated tax, which is $11,239 on $207,524 of CT taxable income plus $680 of add-backs. That fits applying large deductions or credits that Connecticut does not allow at this income."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"It cited itemized deductions in its CT calculation, but Connecticut allows none and taxes CT AGI directly once the personal exemption phases out. Its $7,929 is far below the $11,239 of graduated tax on $207,524 plus $680 of add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It got the $680 of add-backs right. However, it added back $2,852 of tax-exempt interest, which is not a CT addition in this computation, and subtracted 75% of taxable SS ($26,711) instead of taxable SS minus 25% of benefits ($25,140). That produced $208,805 instead of $207,524 and $83 too much tax."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It taxed about $235,031 with no Social Security subtraction and no refund subtraction, leaving out the $25,140 SS adjustment and the $1,750 refund. Its implied phase-out and recapture add-backs exceed the required $680, and its final answer is $2,348 too high."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It applied the schedule to about $234,000 of CT AGI and left out the $25,140 Social Security adjustment (CT caps taxable benefits at 25% for single filers over $75,000) and the $1,750 refund subtraction. That puts about $26,900 of extra income in the 6.5% bracket."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It denied any Social Security relief and taxed $234,414, leaving out the $25,140 SS adjustment and the $1,750 refund subtraction. Its $10,895 is also below the graduated tax on its own base, so neither the base nor the schedule matches the correct $207,524 → $11,239 + $680."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"Its CT taxable income of $200,095 is exactly $7,429 below the correct $207,524, which equals the employer-sponsored insurance premiums. It treated them as an extra wage exclusion, but the $165,597 wage input enters AGI reduced only by the $617 traditional 401(k). Its total also implies $590 of add-backs ($500 + $90) instead of $680 ($250 phase-out + $250 low-tax + $180 middle-tax)."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It taxed about $234,414 with no Social Security or refund subtraction, leaving out the $25,140 SS adjustment and the $1,750 refund. It also applied about $2,250 in total phase-out and recapture add-backs on top of the graduated tax; the required total is $680."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It left out the $25,140 Social Security adjustment. Its $13,286.91 equals the pre-2024 3%/5%/5.5%/6%/6.5% schedule applied to $234,414, with no $680 add-backs. The correct calculation uses 2%/4.5% for the bottom brackets on $207,524 and adds $680."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"Its CT taxable income of $200,095 is exactly $7,429 below the correct $207,524, which matches subtracting the employer-sponsored insurance premiums from wages; AGI uses wages reduced only by the $617 traditional 401(k). It also applied $90 of recapture instead of $180, so add-backs total $590 instead of $680."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"It gave no detailed derivation. The correct figure is $11,239 of graduated tax on $207,524 plus $680 of add-backs. Its $11,779 is $140 short, which fits undercounting those add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It gave no derivation. Its $7,850 is about $4,000 below the correct figure, which fits a much smaller base or phased-out exemptions and credits. The correct figure is $11,239 of graduated tax on $207,524 of CT taxable income plus $680 of add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable Social Security instead of $25,140, since CT keeps 25% of benefits taxable for single filers over $75,000. That gave $197,049 instead of $207,524, and it also left out the $680 of 2% bracket phase-out and recapture add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted all taxable SS and applied a 100% pension exemption, which is fully phased out at this AGI, bringing CT AGI down to $166,833 instead of $207,524. It also used the pre-2024 3%/5% rates and added only $200 toward the required $680 of phase-out and recapture add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-4.7,llm_error,taxable_income_or_deductions,False,"It left out the $25,140 Social Security adjustment and added back $2,852 of tax-exempt interest, giving CT AGI of $235,516 instead of $207,524. It also used the pre-2024 base ($11,050 on $200k, from the 3%/5% rates) and left out the $680 of add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It subtracted all $35,615 of taxable SS instead of $25,140, added back $2,852 of tax-exempt interest, and skipped the $1,750 refund subtraction. It also applied the pre-2024 3%/5% rates and left out the $680 of phase-out and recapture add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It taxed $234,385 with neither the $25,140 Social Security adjustment nor the $1,750 refund subtraction. It also subtracted a $300 property tax credit that is fully phased out at this AGI and left out the $680 of add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value for state_income_tax_before_refundable_credits and no explanation, so there was no answer to grade. The correct value is $11,919.06."
-us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"It returned no value for state_income_tax_before_refundable_credits and no explanation, so there was no answer to grade. The correct value is $11,919.06."
-us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It answered $0 with no reasoning, which implies no Connecticut tax at all. CT taxable income is $207,524 after the $25,140 SS adjustment and $1,750 refund subtraction, giving $11,239 of graduated tax plus $680 of add-backs."
-us,scenario_120,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It taxed about $234,414 with no Social Security or refund subtraction, leaving out the $25,140 SS adjustment and the $1,750 refund. It also used the pre-2024 3% bottom rate."
-us,scenario_120,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"It placed CT taxable income near the correct $207,524 and applied the graduated brackets correctly. However, it left out the $250 2% bracket phase-out add-back and the $430 of low-tax and middle-tax recapture, so its result is roughly the graduated tax alone."
-us,scenario_120,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"It skipped the $25,140 Social Security adjustment and the $1,750 refund subtraction, then invented a CT charitable itemized deduction, which Connecticut does not allow. It also left out the $680 of add-backs. These errors happened to roughly offset, but the base ($218,938) and method do not match the correct $207,524 plus $680."
-us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It left the $35,615 of taxable SS out of federal AGI and then subtracted the full $41,900 of gross benefits again. It also subtracted $6,147 of tax-exempt pension that was never in AGI and applied a $15,000 exemption that is fully phased out at this income, which brought CT taxable income down to $136,825 instead of $207,524."
-us,scenario_120,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"It used a separate-filer rate schedule instead of the single schedule and put CT taxable income at $202,564 instead of $207,524. It then subtracted a $500 property tax credit, which is capped at $300 and fully phased out at this AGI."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model added back $2,852 of tax-exempt interest, which the engine does not add. It treated the Social Security subtraction as vaguely 'limited' instead of computing the exact $25,140 (taxable SS of $35,615 minus 25% of $41,900). It also never itemized the $250 add-back and $430 of recapture. Its CT income of about $205,000 and its $12,130 figure do not come from the correct $207,524 base, which yields $11,239 plus $680."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,state_local_rule,False,"The model got the $25,140 Social Security adjustment right but did not subtract the $1,750 state refund, so its CT AGI was $209,274 instead of $207,524. It added the $250 phase-out add-back and the $180 middle recapture but left out the separate $250 low-tax recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,state_local_rule,False,"The model started from a federal AGI of $236,832, which leaves out the $2,447 farm-rent loss deduction, and it skipped the $1,750 state refund subtraction. That raised CT AGI to $211,692 instead of $207,524. The higher income added 6.5%-bracket tax and pushed middle recapture to $270 instead of $180."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model invented a Connecticut deduction for federal income tax paid and an over-65 deduction, and it applied federal itemized deductions. None of these exist in Connecticut. The correct CT AGI of $207,524 gives $11,239 in bracket tax plus $680 in add-backs, not $0."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"The model gave an unsupported round figure of $7,000. It is far below the correct $11,919: CT AGI of $207,524 (federal AGI less the $25,140 SS adjustment and $1,750 refund), taxed at the 2026 brackets, plus $680 of phase-out and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model said there is no Social Security subtraction above $75,000. In fact Connecticut still subtracts taxable SS above 25% of benefits ($25,140). It also added back tax-exempt interest and skipped the refund subtraction, inflating CT AGI to $236,883. It then cut its own $13,247 bracket result down to $11,890 with no stated computation."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used the pre-2024 3% and 5% low brackets and a loosely estimated base of about $200,000 instead of computing CT AGI as $207,524. Its $9,970 is below even its own bracket math, which gives $11,050 at $200,000. It also omitted the $250 add-back and $430 of recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model claimed the Social Security subtraction phases to zero, yet it used a CT taxable income of about $198,800, which conflicts with that claim. It applied a 2%–5.5% schedule that leaves out the 6% and 6.5% brackets. The correct $207,524 base produces $11,239 in bracket tax plus $680 in add-backs."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,thresholds_rates,False,"The model correctly reached a CT AGI of about $207,495 and a graduated tax of about $11,237. It added the $250 2%-bracket phase-out and the $180 middle recapture. It omitted the separate $250 low-tax recapture, which accounts for the $252 gap."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model wrongly concluded there is no Social Security subtraction above $100,000 of AGI. Connecticut subtracts federally taxable SS above 25% of benefits ($25,140) at any income. The model also skipped the $1,750 refund subtraction, taxed the full $234,414, and omitted the $250 add-back and $430 of recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model subtracted Roth contributions, which are not deductible, and estimated CT AGI at about $198,000. It then reported $7,200, far below what its own 3%–6.99% schedule gives on that base. The correct figure is the 2026 brackets applied to $207,524 ($11,239) plus $680 of phase-out and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model applied the 25%-taxable SS rule correctly but did not subtract the $1,750 state refund, giving CT AGI of $209,891. It added only the $180 middle recapture and left out both the $250 2%-bracket phase-out add-back and the $250 low-tax recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model subtracted all $35,615 of taxable Social Security. Above $75,000 of AGI, Connecticut keeps 25% of benefits ($10,475) taxable, so the correct subtraction is $25,140. The model also added back tax-exempt interest, skipped the refund subtraction, applied a $15,000 personal exemption that is fully phased out at this income, and omitted the $680 in add-backs."
+us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model exempted all Social Security, getting CT AGI of $191,370 instead of $207,524 (the subtraction should be only $25,140). It also took a $1,549 personal tax credit, which phases out completely far below this income, and omitted the $680 in phase-out and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model subtracted only the state refund and skipped the $25,140 Social Security adjustment, leaving CT AGI at $232,664. Its $13,173.16 matches the pre-2024 3%/5% low brackets on that base, and it omitted the add-back and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model subtracted all $35,615 of taxable Social Security instead of $25,140, which leaves 25% of benefits taxable above $75,000 of AGI. That understated CT AGI at $197,049. It also used the pre-2024 3%/5% rates and left out the $250 phase-out add-back and the $250 low-tax recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model inverted the Social Security rule and subtracted only 25% of the taxable amount ($8,904). Connecticut subtracts everything above 25% of total benefits, which is $25,140. That inflated CT AGI to $223,731 instead of $207,524."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model gave no computation, and its $7,800 is far below the correct result. The correct result is CT AGI of $207,524 (after the $25,140 SS adjustment and $1,750 refund subtraction), taxed at the 2026 brackets for $11,239, plus $680 of phase-out and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model assumed high income removes the Social Security subtraction entirely. Connecticut still subtracts taxable SS above 25% of benefits ($25,140). Taxing roughly the full federal AGI pushed the result to $13,272."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model understated federal AGI at $225,235 instead of $234,414. It subtracted 75% of taxable SS ($26,711) instead of taxable SS minus 25% of benefits ($25,140), and it did not subtract the refund. It also added only $150 of recapture and missed the $250 phase-out add-back and the $250 low-tax recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,thresholds_rates,False,"The model gave no computation. Its $11,770 is $149 below the correct $11,919, which is $11,239 of bracket tax on $207,524 plus the full $680 of phase-out add-back, low-tax recapture and middle-tax recapture. The shortfall matches dropping part of those add-backs."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model's CT taxable income of $212,320 is $4,796 too high. It is consistent with adding back tax-exempt interest and not subtracting the $1,750 state refund. The extra base raised the 6.5%-bracket tax even though its $660 of add-backs nearly matched the correct $680."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model gave no derivation. Its $13,125 matches taxing close to the full federal AGI of about $234,000 with no $25,140 Social Security subtraction. The correct CT AGI is $207,524, giving $11,919 after the $680 in add-backs."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,state_local_rule,False,"The model gave no derivation. Its $9,706.75 matches subtracting all taxable Social Security and then taking a personal exemption that is fully phased out. The correct base is $207,524, using a $25,140 SS subtraction, taxed for $11,239 plus $680."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,state_local_rule,False,"The model applied the 75% Social Security exemption to the wrong base and subtracted too much; the correct subtraction is federally taxable SS ($35,615) minus 25% of benefits ($10,475), or $25,140. Its $11,520 is about $400 short, which is consistent with a CT AGI near $201,000 instead of $207,524."
+us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model treated Social Security as fully exempt, but above $75,000 of AGI, 25% of benefits stays taxable. It also left the state refund out of the starting income, invented a $15,000 'standard deduction' that Connecticut does not have, and took a $200 property tax credit that is phased out at this income. It omitted the $680 in add-backs and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,glm-5.3,llm_error,state_local_rule,False,"The model subtracted all $35,615 of taxable Social Security instead of $25,140. It then taxed the $197,049 base at a flat 4.5% instead of the graduated 2%–6.5% schedule, and it skipped the $680 of phase-out and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model gave no computation, and its $4,414 is far too low. Correctly, federal AGI of $234,414 less the $25,140 SS adjustment and $1,750 refund gives CT AGI of $207,524, which produces $11,239 at the 2026 brackets plus $680 of phase-out and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model reduced income with itemized deductions, but Connecticut has no itemized deductions; tax is computed directly on CT AGI. Its $7,929 omits most of the correct tax: $11,239 on $207,524 plus $680 of add-backs."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model's $680 of add-backs matches the engine. Its base is wrong in two ways: it added back $2,852 of tax-exempt interest, and it subtracted 75% of taxable SS ($26,711) instead of taxable SS minus 25% of benefits ($25,140). That made CT income $208,805 instead of $207,524, which adds $83 of 6.5% tax."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model taxed about $235,031 with no Social Security subtraction and no refund subtraction. Connecticut still subtracts $25,140 of SS at this income, and the refund subtraction brings CT AGI down to $207,524. It also overstated the recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model taxed about $234,000 of CT AGI without the $25,140 Social Security subtraction or the $1,750 refund subtraction. That overstated the 6.5% bracket income by about $26,500."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model claimed no Social Security relief at this income, but Connecticut subtracts $25,140, the taxable SS above 25% of benefits. It used $234,414 as CT AGI, yet reported $10,895, which is less than the 2026 brackets give on that base. The correct base is $207,524, giving $11,239 plus $680."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,state_local_rule,False,"The model over-subtracted Social Security and reached CT taxable income of $200,095. The correct subtraction is $25,140 (taxable $35,615 minus 25% of $41,900), giving $207,524. The lower base also cut middle recapture to $90 instead of $180."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,state_local_rule,False,"The model taxed about $234,414 with no Social Security or refund subtraction, then added an inflated recapture. The correct CT AGI is $207,524, and the add-backs total only $680."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,state_local_rule,False,"The model skipped the $25,140 Social Security subtraction. Its $13,286.91 also equals the pre-2024 3%/5% low-bracket schedule applied to $234,414, with no phase-out or recapture, instead of the 2026 2%/4.5% schedule on $207,524."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,state_local_rule,False,"The model over-subtracted Social Security, reaching CT taxable income of $200,095 instead of $207,524. The correct subtraction keeps exactly 25% of the $41,900 benefit taxable, a $25,140 subtraction. The lower base cut the 6.5% bracket tax and halved middle recapture to $90 instead of $180."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,thresholds_rates,False,"The model gave no amounts. Its $11,779 is $140 below the correct $11,919, which is $11,239 of bracket tax on $207,524 plus the full $680 of 2%-bracket phase-out, $250 low-tax recapture and $180 middle-tax recapture. The shortfall matches an incomplete recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model gave no computation. Its $7,850 is far below $11,919, the result of taxing CT AGI of $207,524 (federal AGI less the $25,140 SS adjustment and $1,750 refund) at the 2026 brackets plus $680 of add-backs."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model subtracted all $35,615 of taxable Social Security instead of $25,140, so CT AGI was $197,049 instead of $207,524. It also left out the $250 phase-out add-back and the $250 low-tax recapture, which apply at this income."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model subtracted all taxable Social Security and a 100% pension exemption. The pension exemption is fully phased out at this AGI, and the SS subtraction is limited to $25,140. Together these cut CT AGI to $166,833, and the model also used the pre-2024 3%/5% rates."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"The model added tax-exempt interest and skipped both the $25,140 Social Security subtraction and the $1,750 refund subtraction, so CT AGI was $235,516. It also used the pre-2024 $11,050 tax at $200,000 instead of the 2026 figure of $10,750."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model subtracted all $35,615 of taxable Social Security instead of $25,140, added back tax-exempt interest, and skipped the refund subtraction. It used the pre-2024 3%/5% low brackets and applied none of the $680 of phase-out and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model taxed $234,385 without the $25,140 Social Security subtraction or the $1,750 refund subtraction. It subtracted a $300 property tax credit that is fully phased out at this income and applied none of the $680 of add-backs."
+us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score."
+us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_income_tax_before_refundable_credits, so there is no answer to score."
+us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model reported $0 with no reasoning, which ignores substantial Connecticut tax. CT AGI of $207,524 is taxed at the 2026 brackets for $11,239, plus $680 of phase-out and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,state_local_rule,False,"The model taxed about $233,000 with no Social Security subtraction ($25,140 applies) and no refund subtraction. It also used the pre-2024 3% first-bracket rate instead of 2%."
+us,scenario_120,state_income_tax_before_refundable_credits,muse-wonder,llm_error,thresholds_rates,False,"The model applied the partial SS adjustment correctly and used a base near $208,000, but its $11,300 includes little or none of the $680 in add-backs. The engine adds the $250 2%-bracket phase-out, the $250 low-tax recapture and the $180 middle-tax recapture on top of the $11,239 bracket tax."
+us,scenario_120,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model claimed the Social Security subtraction phases out fully, but $25,140 still applies. It then took a charitable itemized deduction, which Connecticut does not allow. It also skipped the refund subtraction and omitted the $680 of phase-out and recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model left taxable SS out of its federal AGI and then also subtracted the full $41,900 of benefits plus the tax-exempt pension, which is not in AGI to begin with. It applied a $15,000 personal exemption that is fully phased out at this income and used the pre-2024 rates. Together these drove CT taxable income down to $136,825 instead of $207,524."
+us,scenario_120,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model used the married-filing-separately rate schedule for what is a single filer, and it subtracted a $500 property tax credit. That credit is capped at $300 and is fully phased out at CT AGI of $207,524."
us,scenario_120,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_120,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -10624,40 +10740,41 @@ us,scenario_121,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"T
us,scenario_121,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model wrongly treated employer-sponsored insurance premiums as automatically reducing the FICA wage base. The full $25,665 is subject to employee Social Security and Medicare tax."
us,scenario_121,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identified the correct full-wage FICA calculation but submitted $1,911, which does not follow from its own formula. Applying 6.2% and 1.45% to $25,665 produces $1,591.23 and $372.14, totaling $1,963.37; South Carolina adds no employee payroll tax here."
us,scenario_121,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_121,snap,claude-fable-5,llm_error,age_disability,False,"The model treated the `is_disabled` flag as making the household SNAP elderly/disabled, then waived the gross income test and deducted $2,681/month of excess medical costs. Under PolicyEngine's USDA disability definition this household is not elderly/disabled, so the gross test applies. Gross income of $3,326/month exceeds the limit, and SNAP is $0, not $3,540."
-us,scenario_121,snap,claude-fable-5.1,llm_error,age_disability,False,"The model said 'disabled 1-person household: no gross income test' and applied the excess medical deduction to reach $0 net income. The household does not qualify as SNAP-disabled, so the gross income test applies. Gross income of $3,326/month fails it, and the benefit is $0 rather than the $3,576 maximum allotment."
-us,scenario_121,snap,claude-opus-4.7,llm_error,age_disability,False,"The model noted that gross income was above SC's ~$2,510/month BBCE limit, then exempted the household as disabled and applied a medical deduction. The head does not meet the USDA elderly/disabled definition (receipt of SSI, SSDI or a similar program), so failing the gross test ends eligibility and SNAP is $0."
-us,scenario_121,snap,claude-opus-4.8,llm_error,age_disability,False,"The model applied only the net income test because the head is 'disabled' and used large medical deductions to get near-maximum benefits. It also dropped child support from income and produced $4,860, which is above any one-person maximum allotment. The household is not SNAP elderly/disabled, so its $3,326/month gross income fails the gross income test and SNAP is $0."
-us,scenario_121,snap,claude-opus-5.5,llm_error,age_disability,False,"The model said the disabled household 'takes only the net income test' and subtracted $2,682/month in medical costs. PolicyEngine does not treat the head as SNAP-disabled from the generic disability flag and benefits, so the gross income test applies. Gross income of $3,326/month fails it, giving $0 instead of $3,552."
-us,scenario_121,snap,claude-sonnet-4.6,llm_error,age_disability,False,"The model correctly found that $39,909 fails the 130% FPL gross test. It then overrode that result by treating the household as disabled and exempt, and deducted $32,180 of medical costs. The head is not USDA-disabled (no SSI or SSDI receipt), so the failed gross test stands and SNAP is $0."
-us,scenario_121,snap,claude-sonnet-5,llm_error,age_disability,False,"The model correctly saw that gross income exceeds the limit, then exempted the household as elderly/disabled. It also deducted ESI premiums and assumed shelter costs from a mortgage balance with no listed payment. The household does not qualify for the elderly/disabled exemption, so it fails the gross income test and SNAP is $0."
-us,scenario_121,snap,claude-sonnet-5.5,llm_error,age_disability,False,"The model treated the household as disabled, skipped the gross test and applied a medical deduction to reach $0 net income and the $298/month maximum. The head does not meet PolicyEngine's USDA disabled definition, so $3,326/month gross income fails the gross income test and SNAP is $0."
-us,scenario_121,snap,deepseek-v4-flash-0731,llm_error,age_disability,False,"The model applied the uncapped medical deduction and implicitly skipped the gross test, treating the head as a disabled SNAP member. The household is not SNAP elderly/disabled, so its $3,326/month gross income exceeds the gross limit and the benefit is $0, not $3,588."
-us,scenario_121,snap,deepseek-v4-pro-0813,llm_error,age_disability,False,"The model stated the 'disabled household is exempt from the gross income test' and used the excess medical deduction. PolicyEngine does not count the head as USDA-disabled, so the gross income test applies. Income of $3,326/month fails it, giving $0 rather than $3,504."
-us,scenario_121,snap,deepseek-v4.1-flash,llm_error,age_disability,False,"The model exempted the household from the gross income limit because of disability and applied the medical deduction. The head does not meet the USDA elderly/disabled definition, so the gross test applies. $3,326/month gross income fails it and SNAP is $0."
-us,scenario_121,snap,gemini-3-flash-preview,llm_error,age_disability,False,"The model waived the gross income test for a 'household with a disabled member'. It applied an $2,800+ medical deduction plus a shelter deduction even though no shelter costs were listed. The household is not SNAP-disabled, so its $3,326/month gross income fails the gross limit and SNAP is $0."
-us,scenario_121,snap,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model exempted the head from the gross income test as disabled and used medical deductions to zero out net income. Under PolicyEngine's USDA disabled definition the household gets no such exemption, and gross income of $3,326/month exceeds the limit, so SNAP is $0."
-us,scenario_121,snap,gemini-3.5-flash,llm_error,age_disability,False,"The model said the gross income limit is waived because the head is disabled and gave the maximum allotment of $3,504. The head does not qualify as SNAP-disabled, so the gross income test applies. $39,909/year gross income fails it and SNAP is $0."
-us,scenario_121,snap,gemini-3.6-flash,llm_error,age_disability,False,"The model exempted the disabled head from the gross test and took a $2,865/month excess medical deduction. The household does not meet the USDA elderly/disabled definition, so it faces the gross income test. $3,326/month gross income fails it, giving $0."
-us,scenario_121,snap,gemini-3.7-flash,llm_error,age_disability,False,"The model treated the head as disabled for SNAP, waived the gross income test and applied the excess medical deduction. The head is not USDA-disabled, so the gross income test applies. Gross income of $3,326/month exceeds the limit and SNAP is $0, not $3,336."
-us,scenario_121,snap,gemini-4-argon,llm_error,age_disability,False,"The model saw that $39,909 exceeds the 200% FPL BBCE limit but still let the disabled exemption override it and deducted $32,180 of medical costs. The household is not SNAP elderly/disabled under PolicyEngine's USDA definition, so failing the gross test makes it ineligible and SNAP is $0."
-us,scenario_121,snap,glm-5.3,llm_error,age_disability,False,"The model exempted the household from the gross income test as 'with a disabled member' and also dropped child support from gross income. It then applied the medical deduction to reach the maximum allotment. The head does not meet the USDA disabled definition, so gross income above the limit makes SNAP $0."
-us,scenario_121,snap,gpt-5.5,llm_error,age_disability,False,"The model treated the household as a 'one-person disabled SNAP household', which waived the gross test and allowed medical deductions. PolicyEngine does not classify the head as SNAP-disabled, so the gross income test applies. $3,326/month gross income fails it and SNAP is $0."
-us,scenario_121,snap,gpt-5.6-sol,llm_error,age_disability,False,"The model relied on the disabled-household medical deduction to zero out net income and skipped the gross income test. The head is not USDA-disabled, so the household must pass the gross test. At $3,326/month it fails, and SNAP is $0."
-us,scenario_121,snap,gpt-5.6-terra,llm_error,age_disability,False,"The model treated the head as a disabled SNAP member, which waived the gross test and allowed uncapped medical deductions, and gave the $298/month maximum. The household does not meet PolicyEngine's USDA disabled definition, so its gross income fails the gross income limit and SNAP is $0."
-us,scenario_121,snap,gpt-6-astra,llm_error,age_disability,False,"The model applied a $32,180 'disability-related medical deduction' and only a net income test. The head does not qualify as SNAP-disabled, so neither the gross-test exemption nor the medical deduction applies. Gross income of $3,326/month fails the gross limit, giving $0."
-us,scenario_121,snap,gpt-6-luna,llm_error,age_disability,False,"The model applied medical deductions that are available only to elderly/disabled households and skipped the gross income test. The head is not USDA-disabled, so $3,326/month gross income fails the gross test and SNAP is $0."
-us,scenario_121,snap,gpt-6-sol,llm_error,age_disability,False,"The model treated the household as a disabled one-person SNAP unit and used medical deductions to zero out net income. The head does not meet PolicyEngine's USDA disabled definition, so the gross income test applies. $39,909/year gross income fails it and SNAP is $0."
-us,scenario_121,snap,gpt-6.1-sol,llm_error,age_disability,False,"The model said 'disability permits the net-income eligibility test' and deducted $32,180 of medical costs. The head is not SNAP-disabled under the USDA definition, so the gross income test applies. Gross income of $3,326/month exceeds the limit and SNAP is $0."
-us,scenario_121,snap,gpt-6.1-terra,llm_error,age_disability,False,"The model credited the disabled member's medical costs as SNAP deductions and skipped the gross test. The household does not qualify as elderly/disabled for SNAP, so its $3,326/month gross income fails the gross income limit and SNAP is $0."
-us,scenario_121,snap,grok-4.6,llm_error,age_disability,False,"The model said the 'one-person disabled SNAP unit skips the gross-income test' and applied the excess medical deduction. PolicyEngine does not treat the head as USDA-disabled, so the gross test applies. $3,326/month gross income fails it and SNAP is $0."
-us,scenario_121,snap,grok-4.7,llm_error,age_disability,False,"The model waived the gross income test because the head is disabled and deducted $31,780 of medical costs, leaving $488 of net income. The head does not meet the USDA disabled definition, so the gross test applies. $39,909/year gross income fails it and SNAP is $0."
-us,scenario_121,snap,grok-build-0.1,llm_error,age_disability,False,"The model applied 'only net income test' for a disabled head and used medical deductions to reach the maximum allotment. The household is not SNAP elderly/disabled, so its $3,326/month gross income fails the gross income test and SNAP is $0."
-us,scenario_121,snap,inkling,llm_error,age_disability,False,"The model treated the household as a disabled 1-person household, which waived the gross test and allowed out-of-pocket medical deductions. The head does not qualify as SNAP-disabled under PolicyEngine's USDA definition, so gross income above the limit makes SNAP $0."
-us,scenario_121,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no SNAP value or explanation, so there was no parseable answer to score. The correct result is $0, because the household's $3,326/month gross income fails the SNAP gross income test."
-us,scenario_121,snap,kimi-k3,llm_error,age_disability,False,"The model used the head's disability to justify the uncapped medical deduction and never applied the gross income test. It also counted the $2,200 premium amount twice. The head is not USDA-disabled, so the gross test applies. $3,326/month gross income fails it and SNAP is $0."
-us,scenario_121,snap,muse-spark-1.3,llm_error,age_disability,False,"The model treated the household as 'size-1 disabled' and applied the excess medical deduction to qualify for the $292/month maximum. The head does not meet the USDA elderly/disabled definition, so the gross income test applies. At $3,326/month the household fails it and SNAP is $0."
-us,scenario_121,snap,muse-wonder,llm_error,age_disability,False,"The model applied a $2,682/month medical deduction for a 'one-person disabled household' and skipped the gross income test. The household is not SNAP-disabled under PolicyEngine's rules, so its $3,326/month gross income exceeds the gross limit and SNAP is $0."
+us,scenario_121,snap,claude-fable-5,llm_error,categorical_eligibility,False,"Treated the 'is disabled' flag as SNAP disabled status, waived the gross income test, and took a $2,682/month excess medical deduction. The head receives no qualifying benefit such as SSI or Social Security disability, so the household is not elderly/disabled. Its $3,326/month gross income fails South Carolina's 130% FPL gross limit, which makes SNAP $0, not $3,540."
+us,scenario_121,snap,claude-fable-5.1,llm_error,categorical_eligibility,False,"Said a disabled one-person household has no gross income test and deducted about $3,381 in medical costs. Generic disability benefits are not SSI or Social Security disability, so the head does not meet the SNAP definition of disabled. The household faces the 130% FPL gross test, which $3,326/month fails, so the benefit is $0."
+us,scenario_121,snap,claude-fable-5.2,llm_error,categorical_eligibility,False,"Exempted the household from the gross income test because of the disability flag and applied an excess medical deduction. SNAP disabled status requires receipt of a qualifying program such as SSI or Social Security disability, which is not listed. Monthly gross income of about $3,326 exceeds the roughly $1,696 gross limit, so SNAP is $0."
+us,scenario_121,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"Noted that income was above South Carolina's BBCE gross limit but then waived the gross test for a 'disabled' household and deducted $31,780 in medical costs. The head does not qualify as SNAP-disabled because no SSI or Social Security disability is received, so neither the exemption nor the medical deduction applies. The household fails the gross income test and gets $0."
+us,scenario_121,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"Applied only the net income test with a medical deduction, treating the head as disabled, and also left out the $1,200 of child support. Its $4,860 answer is above any one-person maximum allotment. Generic disability benefits do not make the head SNAP-disabled, so the gross income test applies and $3,326/month fails it, giving $0."
+us,scenario_121,snap,claude-opus-5.5,llm_error,categorical_eligibility,False,"Treated the household as disabled, applying only the net income test and a $2,682/month medical deduction. Receiving unspecified 'disability benefits' does not satisfy the SNAP disability definition, which requires SSI, Social Security disability, or a similar benefit. The household therefore fails the 130% FPL gross test and gets $0."
+us,scenario_121,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"Correctly found that $39,909 fails the 130% FPL gross income test, then overrode that result by exempting a 'disabled' household and deducting $32,180 in medical costs. The head receives no SSI or Social Security disability, so the household is not SNAP elderly/disabled. The failed gross income test stands, and SNAP is $0."
+us,scenario_121,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"Applied only the net income test as an 'elderly/disabled' household and deducted medical costs, including the pre-tax ESI premiums. It also invented shelter costs from the mortgage balance. Without SSI or Social Security disability, the head is not SNAP-disabled, so the $39,909 income fails the gross test and the benefit is $0."
+us,scenario_121,snap,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"Assumed a disabled one-person household and used a large medical deduction to bring net income to about $0. The SNAP disabled definition requires receipt of a qualifying program benefit, which generic disability benefits are not. The household's $3,326/month gross income therefore fails the gross limit, and SNAP is $0."
+us,scenario_121,snap,deepseek-v4-flash-0731,llm_error,categorical_eligibility,False,"Gave a 'disabled household' an uncapped medical deduction and skipped the gross income test. The head does not receive SSI or Social Security disability, so the household is not elderly/disabled for SNAP. Gross income of about $3,326/month exceeds South Carolina's 130% FPL limit, so the benefit is $0."
+us,scenario_121,snap,deepseek-v4-pro-0813,llm_error,categorical_eligibility,False,"Exempted the household from the gross income test as disabled and used the excess medical deduction to bring net income to $0. Generic disability benefits do not establish SNAP disability, so the gross test applies. The household fails it with about $3,326/month of income, and SNAP is $0."
+us,scenario_121,snap,deepseek-v4.1-flash,llm_error,categorical_eligibility,False,"Waived the gross income limit for a 'disabled household' and applied medical deductions. The head has no qualifying SSI or Social Security disability receipt, so no exemption applies. The $39,909 annual income fails the 130% FPL gross test, giving $0."
+us,scenario_121,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"Waived the gross income test because of the disability flag, applied a large medical deduction, and also assumed a shelter deduction with no housing costs listed. The head is not SNAP-disabled because no SSI or Social Security disability is received. Monthly income of $3,325 fails the gross limit, so SNAP is $0."
+us,scenario_121,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"Exempted the household from the gross income test because the head is disabled and used medical deductions to bring net income to $0. SNAP disability status requires receipt of a qualifying program benefit, which is absent here. The 130% FPL gross test applies and is failed, so the benefit is $0."
+us,scenario_121,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"Waived the gross income limit for a disabled head and used high medical expenses to cut net income to zero. Generic disability benefits do not meet the SNAP elderly/disabled member definition. Gross income of about $3,326/month exceeds the limit, so SNAP is $0."
+us,scenario_121,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"Treated the head as exempt from the gross income test and deducted $2,865/month in excess medical costs. Without SSI or Social Security disability receipt, the head is not SNAP-disabled, so neither the exemption nor the medical deduction is available. The household fails the gross test and gets $0."
+us,scenario_121,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"Exempted the household from the gross income test as disabled and applied the excess medical deduction. The head receives no qualifying disability program benefit, so the 130% FPL gross test applies. About $3,326/month of income fails it, giving $0."
+us,scenario_121,snap,gemini-4-argon,llm_error,categorical_eligibility,False,"Noticed income was above the broad-based categorical eligibility limit but still exempted a 'disabled' household from the gross test and deducted $32,180 in medical costs. Generic disability benefits are not SSI or Social Security disability, so the household is not elderly/disabled for SNAP. It fails the gross income test, and the benefit is $0."
+us,scenario_121,snap,glm-5.3,llm_error,categorical_eligibility,False,"Treated the household as having a disabled member, which exempts it from the gross test, then used the excess medical deduction to bring net income to zero. It also left out the child support. The head does not meet the SNAP disability definition, which requires SSI, Social Security disability, or similar, so the gross test is failed and SNAP is $0."
+us,scenario_121,snap,gpt-5.5,llm_error,categorical_eligibility,False,"Gave a 'disabled' one-person household the medical-expense deduction and evaluated only net income. Without receipt of a qualifying disability program, the household is not SNAP elderly/disabled. Its $3,326/month gross income fails the 130% FPL gross limit, so SNAP is $0."
+us,scenario_121,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"Deducted medical costs for a 'disabled' household, bringing net income to zero. Generic disability benefits do not establish SNAP disability, so there is no medical deduction and no exemption from the gross test. The household fails the gross income limit, and SNAP is $0."
+us,scenario_121,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"Treated the head as SNAP-disabled and deducted the large medical costs to reach zero net income. The head receives no SSI or Social Security disability, so the 130% FPL gross test applies. About $3,326/month of income fails it, giving $0."
+us,scenario_121,snap,gpt-6-astra,llm_error,categorical_eligibility,False,"Applied a $32,180 'disability-related' medical deduction and only the net income test. The head is not disabled under the SNAP definition, which is based on receipt of a qualifying program such as SSI or Social Security disability. The gross income test applies, $39,909 fails it, and SNAP is $0."
+us,scenario_121,snap,gpt-6-luna,llm_error,categorical_eligibility,False,"Assumed allowable medical deductions brought net income to zero, which requires the household to be SNAP elderly/disabled. Generic disability benefits do not give that status. The household faces the 130% FPL gross test, fails it with about $3,326/month, and gets $0."
+us,scenario_121,snap,gpt-6-sol,llm_error,categorical_eligibility,False,"Treated the head as a disabled SNAP member who can deduct medical expenses and skip the gross test. Without SSI or Social Security disability receipt, neither applies. The $3,326/month gross income exceeds South Carolina's gross limit, so SNAP is $0."
+us,scenario_121,snap,gpt-6.1-sol,llm_error,categorical_eligibility,False,"Explicitly said that 'disability permits the net-income eligibility test' and deducted $32,180 in medical costs. SNAP disabled status requires receipt of a qualifying disability program benefit, which is absent here. The household must pass the gross income test, fails it, and gets $0."
+us,scenario_121,snap,gpt-6.1-terra,llm_error,categorical_eligibility,False,"Treated the head's disability as qualifying for the excess medical deduction and the net-only test. Generic disability benefits are not SSI or Social Security disability, so the household is not SNAP elderly/disabled. It fails the 130% FPL gross test, and SNAP is $0."
+us,scenario_121,snap,grok-4.6,llm_error,categorical_eligibility,False,"Said a one-person disabled SNAP unit skips the gross-income test and applied the excess medical deduction. The head receives no qualifying SSI or Social Security disability, so the gross test applies. About $3,326/month of income fails it, giving $0."
+us,scenario_121,snap,grok-4.7,llm_error,categorical_eligibility,False,"Waived the gross income test for a 'disabled one-person unit' and deducted $31,780 in medical costs. SNAP disability requires receipt of a qualifying disability program benefit, so the head does not qualify. The $39,909 income fails the gross test, and SNAP is $0."
+us,scenario_121,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"Applied only the net income test because the head is disabled and used medical deductions to bring net income to zero. Generic disability benefits do not establish SNAP disability, so the household faces the 130% FPL gross test. Gross income of about $3,326/month fails it, so SNAP is $0."
+us,scenario_121,snap,inkling,llm_error,categorical_eligibility,False,"Gave a 'disabled' household the out-of-pocket medical deduction and evaluated only net income. The head receives no SSI or Social Security disability, so the household is not SNAP elderly/disabled. It fails the gross income limit and gets $0."
+us,scenario_121,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"Returned no SNAP value and no explanation, so there was no answer to score. The correct result is $0, because $3,326/month of gross income fails South Carolina's 130% FPL gross limit for a household that is not SNAP elderly/disabled."
+us,scenario_121,snap,kimi-k3,llm_error,categorical_eligibility,False,"Treated the head as SNAP-disabled, applying the uncapped medical deduction and skipping the gross test. It also double-counted the $2,200 premium. Without SSI or Social Security disability receipt, the head is not SNAP-disabled, so $3,325.75/month of gross income fails the gross limit and SNAP is $0."
+us,scenario_121,snap,muse-spark-1.3,llm_error,categorical_eligibility,False,"Treated the household as disabled, with an excess medical deduction that made net income negative, and never applied the gross income test. Generic disability benefits do not establish SNAP disability, so the 130% FPL gross test applies. The household fails it, and SNAP is $0."
+us,scenario_121,snap,muse-wonder,llm_error,categorical_eligibility,False,"Noted gross income of about $3,326/month but skipped the gross test as a disabled household and deducted about $2,682/month in medical costs. The head receives no qualifying disability program benefit, so neither the exemption nor the deduction applies. The household fails the gross income limit, and SNAP is $0."
us,scenario_121,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_121,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,taxable_income_or_deductions,False,"The model raised the itemized-deduction question and then dismissed it — ""the standard deduction likely exceeds itemized here"" — when the $32,600 of medical outlays less the 7.5%-of-AGI floor leaves about $30,000 of deduction against a $16,100 standard deduction, zeroing federal taxable income and therefore SC's starting point. It then built its $400 on a $14,600 standard deduction (the 2024 federal figure, not 2026's $16,100), a fabricated $4,610 SC personal exemption (SC grants no exemption for the filer, only for dependents), and a flat 6.2% rate in place of SC's 0%/3%/top-rate schedule."
us,scenario_121,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,taxable_income_or_deductions,False,"The model gave no derivation, but $686 is exactly 6.2% × $11,065, i.e. wages of $25,665 reduced by a $14,600 standard deduction and taxed at a single flat rate. That path never claims the $32,600 medical expense itemized deduction, which after the 7.5%-of-AGI floor exceeds the standard deduction by roughly $14,000 and drives SC taxable income to zero, and it also ignores SC's zero-rate bracket on the first ~$3,700 and the 3% bracket above it."
@@ -10717,65 +10834,66 @@ us,scenario_122,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility
us,scenario_122,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that no wage earnings were provided and that payroll tax therefore equals zero, then contradicted that reasoning by imposing an unsupported $1,270.64 tax on unspecified ""taxable income."" Retirement and investment income is not an employee Social Security or Medicare tax base."
us,scenario_122,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable payroll_tax output or explanation, violating the required structured-output contract."
us,scenario_122,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It treated Minnesota's Social Security subtraction as phased out, but at $70,646 AGI the full $27,200 is subtracted. It also claimed itemized deductions beat the standard deduction, when the $14,926 itemized total is below the $19,300 aged/blind standard deduction, so it never took the $4,977 non-itemizer charitable subtraction. Its taxable income of about $52,000 should be $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It used the right structure but overstated two amounts. It used a $19,500 standard deduction instead of $19,300. It set the charitable subtraction at $5,154 (50% of $10,308), when the correct $4,977 also reflects the 0.5%-of-AGI floor on contributions. Together these understated taxable income by $377, giving $18,792 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It cited a $13,175 standard deduction against $43,446 of non-Social-Security income, which leaves more than $30,000 taxable, yet it reported $0 tax. The correct steps give $19,169 of taxable income taxed at 5.35%, which yields $1,025.54."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It cited a 'partially exempt pension' subtraction that Minnesota does not have for private pensions or IRA withdrawals. After the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977 charitable subtraction, $19,169 stays taxable at 5.35%. That is not zero."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It estimated about $1,040 using incorrect deduction amounts and then cut the tax to $0 with a 'senior subtraction' and elderly/disabled credit. Minnesota's elderly/disabled subtraction is fully phased out at $70,646 AGI, so nothing reduces the 5.35% tax on $19,169 of taxable income."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It claimed the Social Security subtraction and deductions push taxable income near zero and invented age-based nonrefundable credits. Subtracting $27,200 Social Security, the $19,300 standard deduction and the $4,977 charitable subtraction from $70,646 AGI still leaves $19,169 taxable."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It treated the Minnesota Social Security subtraction as income-limited and partial, which left taxable income near $44,000. The full $27,200 subtraction applies at this AGI, and with the $19,300 standard deduction and $4,977 charitable subtraction, taxable income is $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct $19,300 standard deduction and full Social Security subtraction. However, it overstated the non-itemizer charitable subtraction at $5,154 (50% of contributions over $500) instead of $4,977, because it left out the 0.5%-of-AGI floor on contributions. That understated taxable income by $177 ($18,992 instead of $19,169)."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It invented a $15,000 Minnesota pension/IRA subtraction for people 65 and older that does not exist. It also started from federal taxable income instead of federal AGI. It then switched to itemizing because it used a $14,575 standard deduction without the two aged/blind add-ons, when the correct $19,300 exceeds itemized deductions."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It asserted that the Social Security subtraction, deductions and a 'disability/elderly credit' bring taxable income to zero. Minnesota AGI after the Social Security subtraction is $43,446, and the $19,300 standard deduction plus the $4,977 charitable subtraction leave $19,169 taxable at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It omitted the non-itemizer charitable subtraction ($4,977), which Minnesota allows on top of the standard deduction. It also used $19,400 instead of $19,300, so taxable income came out at $24,046 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It skipped the full $27,200 Social Security subtraction and the charitable subtraction. It applied a personal exemption that Minnesota no longer allows for the filer, and its $15,200 standard deduction left out the age and blindness add-ons. As a result, taxable income was about $50,146 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It never subtracted the $27,200 of taxable Social Security. It itemized $17,023 instead of taking the larger $19,300 aged/blind standard deduction, and it added a personal exemption Minnesota does not allow for the filer, which pushed taxable income into the 6.8% bracket."
-us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It omitted the $4,977 non-itemizer charitable subtraction and understated the aged/blind standard deduction ($18,500 instead of $19,300). That gave taxable income of $24,946 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It invented a $5,000 retirement subtraction that does not apply to private pension or IRA income, and it used an $18,500 standard deduction instead of $19,300. It also left out the $4,977 non-itemizer charitable subtraction, which happened to roughly offset these errors and land on $19,946 of taxable income instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It applied no Social Security subtraction, so it taxed the full $70,646 AGI less a standard deduction. It also used a fictitious 7.05% second-bracket rate. The full $27,200 subtraction plus the charitable subtraction keep taxable income at $19,169, all in the 5.35% bracket."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The $2,420 answer implies about $42,000 of taxable income spilling into the 6.8% bracket, which fits leaving out the $27,200 Social Security subtraction. The correct taxable income is $19,169 at 5.35%."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It treated the Social Security subtraction as only partial, which put income into the 6.8% bracket. At $70,646 AGI the full $27,200 subtraction applies, and with the $4,977 charitable subtraction taxable income stays at $19,169 in the 5.35% bracket."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"Its taxable income of $51,665 is just AGI minus a standard deduction, so it effectively applied no Social Security subtraction. The full $27,200 subtraction applies at this income, and the $4,977 charitable subtraction also applies."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"Its $1,111 answer implies about $20,766 of taxable income, which is about $1,600 more than the correct $19,169. It understated the combined $19,300 aged/blind standard deduction and $4,977 non-itemizer charitable subtraction taken from Minnesota AGI of $43,446."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It omitted the $4,977 non-itemizer charitable subtraction and used an $18,270 standard deduction instead of $19,300. That gave $25,176 of taxable income instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It submitted $2,389 while its own explanation says $1,308, so the answer contradicts itself. The $1,308 figure also leaves out the $4,977 non-itemizer charitable subtraction applied alongside the $19,300 standard deduction."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It omitted the $4,977 non-itemizer charitable subtraction and used a $19,000 standard deduction instead of $19,300. That left $24,446 of taxable income instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It applied the Social Security subtraction but its $1,365 answer implies about $25,500 of taxable income. That fits leaving out the $4,977 charitable subtraction and understating the $19,300 aged/blind standard deduction."
-us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It applied made-up 2.85%/4.80% rates instead of Minnesota's 5.35% first bracket. It also itemized $14,926 because it used a $14,600 standard deduction without the two aged/blind add-ons; the correct $19,300 standard deduction plus the $4,977 non-itemizer charitable subtraction gives $19,169 of taxable income."
-us,scenario_122,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It omitted the $4,977 non-itemizer charitable subtraction and used an $18,560 standard deduction instead of $19,300. That gave $24,886 of taxable income instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It claimed the standard deduction and senior subtractions wipe out taxable income. After the full $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977 charitable subtraction, $19,169 stays taxable at 5.35%, and no senior subtraction survives at $70,646 AGI."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It reported zero tax with no computation. Minnesota AGI of $43,446 after the Social Security subtraction, less the $19,300 standard deduction and $4,977 charitable subtraction, leaves $19,169 of taxable income and $1,025.54 of tax."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest back to Minnesota income, but no such addition applies here. That pushed its taxable income to about $20,449 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"It said it applied the Social Security subtraction, but its $49,900 taxable income equals AGI less only a standard deduction, so the $27,200 subtraction was never taken. It also left out the $4,977 charitable subtraction, which pushed income into the 6.8% bracket."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest back to Minnesota income, but no such addition applies. That raised taxable income to about $21,055 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"Its $2,640 answer implies about $46,000 of taxable income reaching the 6.8% bracket, which fits leaving out most of the $27,200 Social Security subtraction. It also skipped the $4,977 charitable subtraction; the correct taxable income is $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used a $19,400 standard deduction instead of $19,300. It also used a $5,154 charitable subtraction instead of $4,977, because it left out the 0.5%-of-AGI floor on contributions. That understated taxable income at $18,892 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It omitted the $4,977 non-itemizer charitable subtraction, leaving $23,265 of taxable income. The correct $19,300 standard deduction plus the charitable subtraction from $43,446 gives $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It overstated the combined standard deduction and charitable subtraction at about $25,100 instead of $24,277 ($19,300 plus $4,977). That understated taxable income at $18,342 instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It omitted the $4,977 non-itemizer charitable subtraction and used a $19,400 standard deduction instead of $19,300. That gave $24,046 of taxable income instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"Its $1,284 answer implies about $24,000 of taxable income. That is Minnesota AGI after the Social Security subtraction less only the standard deduction, without the $4,977 non-itemizer charitable subtraction."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It claimed age-based exemptions eliminate taxable income. Minnesota has no filer personal exemption, and no elderly subtraction survives at $70,646 AGI, so $19,169 stays taxable after the Social Security subtraction, the $19,300 standard deduction and the $4,977 charitable subtraction."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It omitted the $4,977 non-itemizer charitable subtraction and used a $19,600 standard deduction instead of $19,300. That gave $23,844 of taxable income instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It left out the non-itemizer charitable subtraction, so it deducted only about $21,400 from Minnesota AGI of $43,446 instead of $24,277 ($19,300 standard deduction plus $4,977). That left about $22,000 taxable instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest back to Minnesota income, but no such addition applies. It also used a $19,400 standard deduction and a $5,154 charitable subtraction instead of $19,300 and $4,977, ending at $20,905 of taxable income instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It applied no Social Security subtraction and itemized $17,023 instead of taking the larger $19,300 aged/blind standard deduction. That gave $53,623 of taxable income instead of $19,169 after the $27,200 Social Security and $4,977 charitable subtractions."
-us,scenario_122,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"It taxed about $50,000 of income because it never subtracted the $27,200 of federally taxable Social Security. It also skipped the $4,977 non-itemizer charitable subtraction, which pushed income into the 6.8% bracket."
-us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model gave no value for state_income_tax_before_refundable_credits, so there was no answer to score."
-us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It subtracted a $5,330 personal exemption, which Minnesota no longer allows for the filer. It also omitted the $4,977 non-itemizer charitable subtraction, and the two errors largely offset each other. The $19,425 standard deduction also overstates the correct $19,300."
-us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It applied a 'senior age-65 subtraction' and concluded taxable income is zero. The elderly subtraction is fully phased out at $70,646 AGI, and after the $27,200 Social Security subtraction, the $19,300 standard deduction and the $4,977 charitable subtraction, $19,169 stays taxable."
-us,scenario_122,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It omitted the $4,977 non-itemizer charitable subtraction and used an $18,200 standard deduction instead of $19,300. That gave $25,246 of taxable income instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It omitted the $4,977 non-itemizer charitable subtraction and understated the aged/blind standard deduction at about $17,950 instead of $19,300. That gave $25,496 of taxable income instead of $19,169."
-us,scenario_122,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It itemized $17,367 because it compared against a $15,300 standard deduction without the age and blindness add-ons. The correct $19,300 standard deduction exceeds itemized deductions and also unlocks the $4,977 non-itemizer charitable subtraction, giving $19,169 of taxable income instead of $26,079."
-us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It never subtracted the $27,200 of taxable Social Security and added back tax-exempt interest and pension income. It used a $14,600 standard deduction without the age and blindness add-ons. It then treated the refundable property tax refund as a reduction to tax before refundable credits."
-us,scenario_122,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used fabricated amounts: AGI of $52,092 instead of $70,646 and a $30,100 standard deduction instead of $19,300. It also skipped the $27,200 Social Security subtraction and the $4,977 charitable subtraction. Its own $21,992 at 5.35% comes to $1,177, not the $1,450 it reported."
-us,scenario_122,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model computed household income at about $77,500, noted it was 'near the income limit' and gave $2,265, more than its own $1,470 estimate, though it said the refund would be reduced. The engine puts renter's-credit household income at $65,346, inside the schedule, but requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0."
-us,scenario_122,state_refundable_credits,deepseek-v4.1-flash,llm_error,credit_phaseout,False,"The model used $77,459 of household income and a '50% bracket with 2.5% threshold' to estimate $1,431.74. The engine uses federal AGI with an age-based subtraction for this filer, giving $65,346, inside the schedule, but requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0."
-us,scenario_122,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model assumed the head qualifies for the renter's credit and gave $2,000. The engine requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0."
-us,scenario_122,state_refundable_credits,gemini-4-argon,llm_error,credit_phaseout,False,"The model based its $1,140 renter's credit on the high rent of $28,235. The engine requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0."
-us,scenario_122,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model applied a 'high-income copayment' to 17% of rent and assumed some credit remains. The engine puts renter's-credit household income at $65,346, inside the schedule, but requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the state refundable credit total is $0."
-us,scenario_122,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model applied a senior/disabled adjustment, placed the household inside the renter's credit schedule and gave $2,228. The engine also applies an age-based household-income subtraction and places the household inside the schedule, but it requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit and total refundable credits are $0."
-us,scenario_122,state_refundable_credits,gpt-6-astra,llm_error,credit_phaseout,False,"The model started from federal AGI of $70,646 and applied an elderly/disabled subtraction. The engine also applies a $5,300 age-based subtraction, giving renter's-credit household income of $65,346. The engine requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0."
-us,scenario_122,state_refundable_credits,gpt-6.1-terra,llm_error,credit_phaseout,False,"The model gave a $300 renter's credit from rent, household income and the senior income adjustment. The engine requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0. No other MN refundable credit applies, because there is no earned income and there are no children."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"It assumed the Minnesota Social Security subtraction phases out at about $70k AGI, so the $27,200 of taxable benefits stayed in income. It also itemized instead of taking the $19,300 aged/blind standard deduction plus the $4,977.18 non-itemizer charitable subtraction. With the full subtraction, taxable income is $19,169, taxed only at 5.35%, not about $52,000 reaching the 6.8% bracket."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"The structure was right: full Social Security subtraction, charitable subtraction and aged/blind standard deduction. However, it overstated the standard deduction ($19,500 instead of $19,300) and the charitable subtraction ($5,154 instead of $4,977.18). That cut taxable income to $18,792 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5.2,llm_error,taxable_income_or_deductions,False,"It used a $19,400 standard deduction instead of $19,300 and a $5,154 charitable subtraction instead of $4,977.18. Taxable income came out at $18,892 instead of $19,169, so the 5.35% tax was about $15 too low."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It asserted $0 without doing the arithmetic. Even its own $43,446 of non-Social Security income minus a $13,175 deduction leaves about $30,000 taxable. The correct path subtracts the $19,300 standard deduction and $4,977.18 charitable subtraction from Minnesota AGI of $43,446, leaving $19,169 taxed at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-5.5,llm_error,state_local_rule,False,"It relied on a 'partially exempt pension' subtraction. Minnesota has no general subtraction for private pension or IRA income. After the $27,200 Social Security subtraction, $19,300 standard deduction and $4,977.18 charitable subtraction, $19,169 is still taxable at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"It correctly reached about $19,442 of taxable income and about $1,040 of tax, then cut that to $0 with Minnesota's elderly/disabled subtraction and a long-term care credit. The filer's $70,646 AGI and $32,000 of Social Security far exceed that subtraction's income limits, and no such credit applies."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"It claimed nonrefundable age-related credits and deductions push taxable income to about zero. Minnesota has no such credit. The Social Security subtraction, $19,300 standard deduction and $4,977.18 charitable subtraction still leave $19,169 taxable at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"It treated the Minnesota Social Security subtraction as partly limited and left about $44,000 taxable. At $70,646 single AGI the full $27,200 is subtracted. It also used a standard deduction of about $18,250 instead of $19,300 and left out the $4,977.18 charitable subtraction."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"The Social Security subtraction and $19,300 standard deduction were right, but it overstated the non-itemizer charitable subtraction as $5,154 (50% of $10,808 minus $500) instead of $4,977.18. Taxable income was $18,992 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"It invented a $15,000 Minnesota pension/IRA subtraction for filers 65 and older; Minnesota has none for private pensions or IRAs. It also itemized with the full $3,510 of employee expenses instead of taking the $19,300 aged/blind standard deduction plus the $4,977.18 charitable subtraction. Taxable income fell to $10,010 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"It claimed itemized deductions, age/blind amounts and a Minnesota elderly/disability credit wipe out taxable income. The elderly/disabled relief is income-limited and unavailable at $70,646 AGI. The correct deductions leave $19,169 taxable at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5.5,llm_error,taxable_income_or_deductions,False,"It omitted the Minnesota non-itemizer charitable subtraction ($4,977.18) and used a $19,400 standard deduction instead of $19,300. Taxable income was $24,046 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It skipped the full $27,200 Social Security subtraction and used a $15,200 standard deduction without the aged/blind additions. It also subtracted a personal exemption, which Minnesota allows only for dependents, and dropped the charitable subtraction. Taxable income of $50,146 wrongly reached the 6.8% bracket."
+us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"It never applied the full $27,200 Minnesota Social Security subtraction. It itemized $17,023 and took a $5,094 filer personal exemption that Minnesota does not allow, instead of the $19,300 aged/blind standard deduction plus $4,977.18 charitable subtraction. Taxable income was $48,529 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It understated the aged/blind standard deduction ($18,500 instead of $19,300) and omitted the $4,977.18 non-itemizer charitable subtraction. Taxable income was $24,946 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It invented a $5,000 retirement subtraction; Minnesota has none for private pension or IRA income. It also used an $18,500 standard deduction instead of $19,300 and left out the $4,977.18 charitable subtraction, producing $19,946 of taxable income instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"It taxed full federal AGI without the $27,200 Social Security subtraction or the $4,977.18 charitable subtraction. It also used $18,700 instead of the $19,300 standard deduction and applied 7.05% instead of Minnesota's 6.8% second-bracket rate."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It gave no computation. $2,420 implies about $42,500 of taxable income reaching the 6.8% bracket, which fits leaving out most of the full $27,200 Social Security subtraction. The correct deductions leave $19,169 taxed at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"It applied only a partial Social Security subtraction, so taxable income of about $51,000 reached the 6.8% bracket. At $70,646 single AGI the filer gets the full $27,200 subtraction, and the $4,977.18 charitable subtraction also applies."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"It treated the Social Security subtraction as partial and reached $51,665 of taxable income, which effectively keeps the $27,200 of taxable benefits. The filer qualifies for the full subtraction, and adding the $19,300 standard deduction and $4,977.18 charitable subtraction gives $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no computation. $1,111 at 5.35% implies about $20,766 of taxable income, roughly $1,600 above the correct $19,169. That fits understating the $19,300 aged/blind standard deduction or the $4,977.18 charitable subtraction."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It used an $18,270 standard deduction instead of $19,300 and omitted the $4,977.18 non-itemizer charitable subtraction. Taxable income was $25,176 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its submitted value of $2,389 contradicts its own explanation, which ends 'value = 1308'. The $1,308 path also omits the $4,977.18 non-itemizer charitable subtraction and uses a $19,000 standard deduction instead of $19,300."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It used a $19,000 standard deduction instead of $19,300 and omitted the $4,977.18 non-itemizer charitable subtraction. Taxable income was $24,446 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-4-argon,llm_error,taxable_income_or_deductions,False,"It applied the full $27,200 Social Security subtraction, but $1,365 implies about $25,500 of taxable income. That means it omitted the $4,977.18 charitable subtraction and understated the $19,300 aged/blind standard deduction."
+us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It itemized $14,926 on the belief that the Minnesota standard deduction is $14,600, ignoring the aged and blind additions. The correct $19,300 standard deduction is larger and also unlocks the $4,977.18 charitable subtraction. It then applied 2.85%/4.80% rates, which are not Minnesota's; the bottom rate is 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,glm-5.3,llm_error,taxable_income_or_deductions,False,"It reached the right Minnesota AGI of $43,446, but used about $18,560 instead of the $19,300 standard deduction and omitted the $4,977.18 non-itemizer charitable subtraction. Taxable income was $24,886 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"It claimed senior/disabled subtractions eliminate taxable income. Minnesota's elderly/disabled subtraction has income limits that this filer's $70,646 AGI and $32,000 of Social Security far exceed. The correct deductions leave $19,169 taxed at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted non-positive tax without any computation. Minnesota AGI of $43,446 (after the $27,200 Social Security subtraction), minus the $19,300 standard deduction and $4,977.18 charitable subtraction, leaves $19,169 taxed at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest as a Minnesota addition, which the reference does not include. Its $1,094 implies about $20,449 of taxable income instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"Its taxable income of about $49,900 means it did not subtract the full $27,200 of taxable Social Security, for which the filer qualifies at $70,646 AGI. It also omitted the $4,977.18 charitable subtraction, so income wrongly reached the 6.8% bracket."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest back as a Minnesota addition, which the reference does not include. Taxable income came out at $21,055 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"$2,640 implies about $45,800 of taxable income spilling into the 6.8% bracket. That fits applying only part of the $27,200 Social Security subtraction and skipping the $4,977.18 charitable subtraction. The full subtractions leave $19,169 taxed at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It used a $19,400 standard deduction instead of $19,300 and a $5,154 charitable subtraction instead of $4,977.18. Taxable income was $18,892 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-luna,llm_error,taxable_income_or_deductions,False,"It applied the Social Security subtraction and the aged/blind standard deduction, but omitted the $4,977.18 non-itemizer charitable subtraction. Taxable income was about $23,265 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"Its $18,342 taxable income is $827 below the correct $19,169. It overstated the combined standard deduction and charitable subtraction, whose correct values are $19,300 and $4,977.18."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It omitted the $4,977.18 non-itemizer charitable subtraction and used a $19,400 standard deduction instead of $19,300. Taxable income was $24,046 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-6.1-terra,llm_error,taxable_income_or_deductions,False,"It applied the Social Security subtraction and standard deduction but omitted the $4,977.18 non-itemizer charitable subtraction. $1,284 implies about $24,000 of taxable income instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"It claimed age-based exemptions bring taxable income to zero. Minnesota gives no extra age exemption beyond the aged/blind standard deduction additions already in the $19,300 deduction, so $19,169 is still taxable at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It used a $19,600 standard deduction instead of $19,300 and omitted the $4,977.18 non-itemizer charitable subtraction. Taxable income was $23,844 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It subtracted Social Security and took the standard deduction, but did not take the full $4,977.18 non-itemizer charitable subtraction. Taxable income was about $22,000 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-4.7,llm_error,state_local_rule,False,"It added the $2,013 of tax-exempt interest as a Minnesota addition, which the reference does not include. It also used a $19,400 standard deduction and a $5,154 charitable subtraction instead of $19,300 and $4,977.18, giving $20,905 of taxable income instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"It never subtracted the $27,200 of taxable Social Security. It also itemized $17,023 instead of taking the $19,300 aged/blind standard deduction plus the $4,977.18 charitable subtraction, leaving $53,623 taxable in the 6.8% bracket instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"Its taxable income of about $50,000 leaves the $27,200 of taxable Social Security in Minnesota income. The filer qualifies for the full Social Security subtraction at $70,646 AGI, which with the deductions brings taxable income to $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for this output, so no answer could be scored."
+us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"It subtracted a $5,330 personal exemption for the filer, but Minnesota's exemption applies only to dependents. It also omitted the $4,977.18 non-itemizer charitable subtraction and used a $19,425 standard deduction instead of $19,300. Taxable income was $18,690 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"It invented a 'senior age-65 subtraction' and assumed Social Security was only partly taxable. Federal AGI includes $27,200 of Social Security, which Minnesota subtracts, and after the $19,300 standard deduction and $4,977.18 charitable subtraction, $19,169 is still taxable."
+us,scenario_122,state_income_tax_before_refundable_credits,muse-spark-1.3,llm_error,taxable_income_or_deductions,False,"It used an $18,200 standard deduction instead of $19,300 and omitted the $4,977.18 non-itemizer charitable subtraction. Taxable income was about $25,246 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,muse-wonder,llm_error,taxable_income_or_deductions,False,"It applied the Social Security subtraction but deducted only about $17,950 and omitted the $4,977.18 non-itemizer charitable subtraction. Taxable income was $25,496 instead of $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It compared $17,367 of itemized deductions against a $15,300 standard deduction that ignores the aged and blind additions. The correct $19,300 standard deduction is larger and adds the $4,977.18 charitable subtraction, so its $26,079 of taxable income should have been $19,169."
+us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"It skipped the $27,200 Social Security subtraction and added back tax-exempt interest and pension income. It used a $14,600 standard deduction without aged/blind additions, then arbitrarily lowered the tax for the property tax refund, which is a separate refundable credit."
+us,scenario_122,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It used a federal AGI of $52,092 instead of $70,646 and a made-up $30,100 standard deduction. Its $1,450 tax also does not match its own $21,992 taxable income, which at 5.35% gives about $1,177."
+us,scenario_122,state_refundable_credits,claude-fable-5,prompt_ambiguity,state_local_rule,False,"The model computed household income at about $77,500, noted it was 'near the income limit' and gave $2,265, more than its own $1,470 estimate, though it said the refund would be reduced. The engine puts renter's-credit household income at $65,346, inside the schedule, but requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0."
+us,scenario_122,state_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,state_local_rule,False,"The model used $77,459 of household income and a '50% bracket with 2.5% threshold' to estimate $1,431.74. The engine uses federal AGI with an age-based subtraction for this filer, giving $65,346, inside the schedule, but requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0."
+us,scenario_122,state_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,state_local_rule,False,"The model assumed the head qualifies for the renter's credit and gave $2,000. The engine requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0."
+us,scenario_122,state_refundable_credits,gemini-4-argon,prompt_ambiguity,state_local_rule,False,"The model based its $1,140 renter's credit on the high rent of $28,235. The engine requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0."
+us,scenario_122,state_refundable_credits,gpt-5.5,prompt_ambiguity,state_local_rule,False,"The model applied a 'high-income copayment' to 17% of rent and assumed some credit remains. The engine puts renter's-credit household income at $65,346, inside the schedule, but requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the state refundable credit total is $0."
+us,scenario_122,state_refundable_credits,gpt-5.6-sol,prompt_ambiguity,state_local_rule,False,"The model applied a senior/disabled adjustment, placed the household inside the renter's credit schedule and gave $2,228. The engine also applies an age-based household-income subtraction and places the household inside the schedule, but it requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit and total refundable credits are $0."
+us,scenario_122,state_refundable_credits,gpt-6-astra,prompt_ambiguity,state_local_rule,False,"The model started from federal AGI of $70,646 and applied an elderly/disabled subtraction. The engine also applies a $5,300 age-based subtraction, giving renter's-credit household income of $65,346. The engine requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0."
+us,scenario_122,state_refundable_credits,gpt-6.1-terra,prompt_ambiguity,state_local_rule,False,"The model gave a $300 renter's credit from rent, household income and the senior income adjustment. The engine requires a qualifying Certificate of Rent Paid (CRP), whose unlisted input defaults to false, so the renter's credit is $0. No other MN refundable credit applies, because there is no earned income and there are no children."
us,scenario_122,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for state_refundable_credits, so there was nothing to score. The correct derivation gives $0, because the required qualifying Certificate of Rent Paid (CRP) input is unlisted and defaults to false and no other MN refundable credit applies."
us,scenario_123,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model gave no child1_chip_eligible value and no explanation, so its output is missing rather than a wrong eligibility call. The correct derivation is 0: household MAGI of about $145,002, which includes the child's $45,000 of wages, is over 500% FPL and far above Pennsylvania's 314% FPL CHIP ceiling."
us,scenario_123,child1_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model counted Pennsylvania's paid CHIP for higher-income families as eligibility and never applied the CHIP income ceiling. PolicyEngine's CHIP eligibility requires household MAGI at or below Pennsylvania's 314% FPL limit. This household is at about $145,002, over 500% FPL for three people, because the child's $45,000 of wages counts in MAGI. The child is therefore ineligible (0), not 1."
@@ -10786,56 +10904,57 @@ us,scenario_123,child1_head_start_eligible,glm-5.2,parse_contract_failure,missin
us,scenario_123,child1_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,child1_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,child1_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It omitted the 2026 $2,000 non-itemizer charitable deduction, which put the parents' taxable income at $67,802 instead of $65,802. It then tried to add the child's separate-return tax of $3,220 under the adjudicated alternative reading that sums household returns. The submitted $5,860 matches neither of its own subtotals ($5,440 or $8,660)."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It put the dependent child's $45,000 of wages into the joint tax-unit AGI ($145,002), but the parents' AGI is only $100,002. Its deductions and CTC were right, but the extra $45,000 was taxed at 12–22%, which gave $11,600.44 instead of $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It imputed about $25,200 of mortgage interest from the $420,000 balance, although unlisted interest is $0, and itemized on that basis. It also used a $29,200 standard deduction instead of $32,200, counted two CTC children instead of one, and missed the $2,000 non-itemizer charitable deduction. The $8,316 it submitted does not follow from its own work."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the household's combined $145,000 of wages, including the dependent child's $45,000, on the joint return, when the tax unit's AGI is $100,002. It also left out the $2,000 non-itemizer charitable deduction and applied no child tax credit, even though the $2,200 CTC is fully usable against the parents' liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It used a $31,500 standard deduction (the 2025 figure) instead of the 2026 $32,200 and omitted the $2,000 non-itemizer charitable deduction. It also used a $2,000 CTC instead of $2,200, then replaced its own $5,724 result with an unexplained $5,316."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It made up about $29,400 of mortgage interest by applying 7% to the $420,000 balance and itemized, when the unlisted interest is $0 and the standard deduction plus the $2,000 charitable deduction applies. It also wrongly denied the $2,200 CTC under the support test; for a qualifying child, that test only asks whether the child provides over half of their own support. The final $12,053 is not derived."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It used the correct $32,200 standard deduction and $2,200 CTC but omitted the 2026 $2,000 non-itemizer charitable deduction, which left taxable income at $67,802 instead of $65,802. It also understated bracket tax on that income ($7,610 vs. $7,640.24), and ended at $5,410 instead of $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent child's $45,000 of wages in joint AGI ($145,002) when the parents' tax-unit AGI is $100,002. That raised taxable income to $110,802 instead of $65,802; its deductions and CTC were otherwise correct."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It projected 2026 parameters under extended TCJA: a $30,750 standard deduction, a $2,000 CTC, and indexed 2025 brackets. It did not use the enacted OBBBA values: a $32,200 standard deduction, a $2,200 CTC, and a 10% bracket ending at $24,800. It also omitted the $2,000 non-itemizer charitable deduction."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It chose $18,557 of itemized deductions while stating that this amount exceeded the $32,200 standard deduction, which is false. That gave taxable income of $81,445 instead of $65,802 after the standard deduction and $2,000 charitable deduction. It then submitted $15,300, which does not match its own $7,300 estimate."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed a $13,650 qualified-overtime deduction for the head. The engine uses an explicit FLSA overtime-premium input, which is $0 when unlisted. It also added the child's separate $3,220 tax under the adjudicated alternative reading that sums household returns. The output is excluded because the prompt leaves return scope unspecified."
-us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in joint AGI ($145,002 instead of $100,002). It also used a $30,800 standard deduction instead of $32,200, omitted the $2,000 non-itemizer charitable deduction, and applied a $2,000 CTC instead of $2,200."
-us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA sunset rules (personal exemptions, 15% bracket, $1,000 CTC) even though OBBBA made the TCJA structure permanent for 2026. It also added $4,979.25 of tax from the child's separate return to the joint tax-unit liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It used pre-TCJA 2026 rules (three $5,300 personal exemptions, 15% bracket, $1,000 CTC) instead of OBBBA's $32,200 standard deduction, $2,000 charitable deduction, and $2,200 CTC. It also added the child's separate-return tax of $4,896.25 to the parents' tax-unit liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA had expired: it itemized $18,557, took $15,900 of personal exemptions, used the 15% bracket, and applied a $1,000 CTC. It also added the child's separate-return tax of $4,855 under the adjudicated alternative reading that sums household returns."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA had expired, imputed $16,800 of mortgage interest from the balance although unlisted interest is $0, and used a $1,000 CTC. It then added the child's separate $4,955 tax to the parents' liability under the adjudicated alternative reading that sums household returns."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It used $145,002 of joint AGI, including the dependent child's $45,000 of wages, when the tax unit's AGI is $100,002. It never applied the $32,200 standard deduction plus the $2,000 charitable deduction and the $2,200 CTC to arrive at $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA 2026 rules (personal exemptions, itemizing, $1,000 CTC) instead of OBBBA's permanent $32,200 standard deduction and $2,200 CTC. It also added the child's separate $4,925 tax to the joint tax-unit liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA had expired, with three $5,050 exemptions, the 15% bracket, and a $1,000 CTC. It then added the child's separate $5,023 tax to the parents' $7,784 under the adjudicated alternative reading that sums household returns."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. The correct path is $100,002 minus the $32,200 standard deduction and the $2,000 non-itemizer charitable deduction, which gives $65,802 of taxable income, $7,400.24 of tax, and $5,200.24 after the $2,200 CTC. Its $5,304 is consistent with understating deductions by roughly $860."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's $45,000 of wages into joint AGI ($145,002) and applied pre-TCJA personal exemptions and itemizing. It then treated the CTC as phased out, although at the parents' $100,002 AGI the full $2,200 CTC applies with no phase-out."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It used $145,002 of AGI including the dependent's wages and applied pre-TCJA personal exemptions with $18,557 of itemized deductions. The correct approach applies OBBBA's $32,200 standard deduction plus the $2,000 charitable deduction to $100,002, then subtracts the $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in joint AGI and used 2025 parameters: a $30,000 standard deduction and 2025 brackets. It also omitted the $2,000 non-itemizer charitable deduction and used a $2,000 CTC instead of $2,200."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly limited AGI to $100,002 but omitted the 2026 $2,000 non-itemizer charitable deduction. It subtracted a $2,000 CTC instead of the $2,200 OBBBA credit, which left it about $536 above $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It used a projected $30,750 standard deduction and a $2,000 CTC instead of OBBBA's $32,200 and $2,200, and it omitted the $2,000 charitable deduction. It then added the child's separate $3,312.08 tax to the joint tax-unit liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in joint AGI and stacked a $16,100 dependent standard deduction onto the joint return. It also treated the CTC as fully refundable and so applied no nonrefundable CTC, although the $2,200 credit is absorbed against the $7,400.24 liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It counted $145,001 of wages, including the dependent child's $45,000, on the joint return, which produced about $116,751 of taxable income. The correct taxable income is $65,802, after taking the $32,200 standard deduction and $2,000 charitable deduction from $100,002."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. Its $14,677 is consistent with taxing the combined $145,000 of wages, including the dependent child's, on the joint return. The tax unit's AGI is $100,002, and after the $34,200 of deductions and the $2,200 CTC the result is $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's $45,000 of wages into joint AGI ($145,002 instead of $100,002) and omitted the $2,000 non-itemizer charitable deduction. That produced $112,802 of taxable income instead of $65,802."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent child's $45,000 of wages in joint AGI ($145,002). Its deductions and $2,200 CTC were correct, but the tax-unit AGI is $100,002, which yields $65,802 of taxable income."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the household's combined wages, including the child's $45,000, on the joint return using a 2025 $30,000 standard deduction and 2025 brackets. It also omitted the $2,000 non-itemizer charitable deduction; the 2026 values are a $32,200 standard deduction and a 10% bracket up to $24,800."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"It computed the parents' joint liability exactly ($5,200.24) and added the dependent child's separate-return tax of $3,220. Its $8,420.24 total is the recorded alternative value, and the output is excluded because return scope is unspecified."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It omitted the 2026 $2,000 non-itemizer charitable deduction, which put taxable income at $67,802 instead of $65,802. It also added the child's separate $3,220 tax to the joint tax-unit liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It took a $13,650 qualified-overtime deduction for the head, who is not flagged as paid hourly and gets none, and it omitted the $2,000 non-itemizer charitable deduction. It also added the child's separate $3,220 tax to the joint unit's liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It derived the joint return's $5,200.24 correctly and added $3,220 for the dependent child's separate wage return. Its $8,420.24 total is the recorded alternative value, and the output is excluded because return scope is unspecified."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It used household-wide wages, including the child's $45,000, as joint AGI ($145,002) and omitted the $2,000 non-itemizer charitable deduction. That produced $12,040.44 instead of taxing $65,802 to reach $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It combined the head's and the child's wages into a $145,002 joint AGI and deducted mortgage interest that is not in the inputs ($0 when unlisted). The correct approach applies the $32,200 standard deduction plus the $2,000 charitable deduction to $100,002, then subtracts the $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA sunset rules (personal exemptions, 15% bracket, $1,000 CTC) instead of OBBBA's permanent $32,200 standard deduction and $2,200 CTC. It also added about $4,851 of tax from the child's separate return to the joint unit's liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's wages into joint AGI ($145,002) and applied pre-TCJA exemptions and 15%/25% brackets. It also treated the CTC as phased out above an unindexed $110,000 threshold, although the 2026 MFJ threshold is $400,000 and the full $2,200 CTC applies."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent child's $45,000 of wages in joint AGI and assumed the TCJA had expired, with $5,400 personal exemptions and 15%/25% brackets. It also eliminated the CTC with the pre-TCJA $110,000 phase-out, although OBBBA provides a $2,200 CTC with a $400,000 threshold."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA rules (three $5,434 exemptions, a 10% medical floor, 15% bracket, $1,000 CTC) instead of OBBBA's 2026 law. It also added the child's separate $4,846 tax to the joint tax-unit liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It reached roughly the correct joint-return liability (about $5,210, vs. the exact $5,200.24) and added the child's separate-return tax of about $3,225 under the adjudicated alternative reading that sums household returns."
-us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It produced no usable answer or explanation for federal_income_tax_before_refundable_credits, so no value could be scored."
-us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in joint AGI ($145,002) and invented a $12,000 qualified-overtime deduction for a head who is not flagged as paid hourly. It also omitted the $2,000 non-itemizer charitable deduction."
-us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed nonrefundable credits wipe out the liability. The $2,200 CTC only reduces the $7,400.24 of regular tax on $65,802 of taxable income to $5,200.24, so the result is not $0. It also confused refundable credits with the before-refundable-credits measure."
-us,scenario_123,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly limited AGI to $100,002 and used the $32,200 standard deduction and $2,200 CTC, but omitted the 2026 $2,000 non-itemizer charitable deduction. Its taxable income was therefore $67,802 instead of $65,802, and it came out $244 too high."
-us,scenario_123,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in joint AGI and took a $13,650 overtime-premium deduction for the head, who is not paid hourly and gets none. It also used a $31,500 standard deduction instead of $32,200."
-us,scenario_123,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's wages in joint AGI ($145,002) and applied a $12,500 OBBBA overtime deduction for the head, who is not flagged as paid hourly. It also omitted the $2,000 non-itemizer charitable deduction."
-us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in AGI, imputed $27,300 of mortgage interest from the balance (unlisted interest is $0), and itemized. It also used 2025 brackets and a $2,000 CTC instead of 2026's $32,200 standard deduction plus $2,000 charitable deduction and the $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It made up deductions and credits that do not apply: a QBI deduction on W-2 wages, imputed mortgage interest, a CDCC with no care expenses, and a foreign earned income exclusion. With these it zeroed out a liability that is $7,400.24 before the $2,200 CTC and $5,200.24 after it."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It computed taxable income as $67,802 and never subtracted the 2026 $2,000 non-itemizer charitable deduction for joint filers, which brings taxable income to $65,802. It then submitted $5,860, a number that matches neither its own parents-only result ($5,440) nor its combined figure with the child ($8,660). The correct parents-only liability is $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It put the dependent child's $45,000 of wages into the joint AGI ($145,002), but the parents' unit AGI is only $100,002. That inflated taxable income from $65,802 to $110,802 and pushed part of it into the 22% bracket. The rest of its method (the $32,200 standard deduction, the $2,000 non-itemizer charitable deduction and the $2,200 CTC) was correct."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in the joint AGI ($145,002 instead of $100,002). It also took a $12,000 qualified overtime deduction for the head, who is not flagged as paid hourly and gets no overtime deduction. Both mistakes moved taxable income away from the correct $65,802."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It imputed $25,200 of mortgage interest from the $420,000 balance, although no interest paid is listed, so that input is zero. It also used an outdated $29,200 standard deduction and counted two CTC children instead of one. Its final $8,316 does not follow from any of its intermediate figures, whereas $32,200 standard plus $2,000 charitable on $100,002 AGI gives $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the combined $145,000 of wages, including the dependent child's $45,000 that is not in the parents' AGI. It also ignored the $2,200 CTC and the $2,000 non-itemizer charitable deduction. Its result was a rough $14,500 instead of tax on $65,802 of taxable income less the CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It used the 2025 standard deduction of $31,500 instead of the 2026 figure of $32,200, left out the $2,000 non-itemizer charitable deduction, and applied a $2,000 CTC instead of $2,200. It then arbitrarily adjusted its computed $5,724 down to $5,316 rather than reaching $65,802 taxable and $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It made up $29,400 of mortgage interest from the loan balance, although unlisted interest paid is zero, and so itemized wrongly. It also denied the CTC for the 16-year-old, who is a qualifying dependent worth $2,200. Its final $12,053 has no derivation."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It left out the 2026 $2,000 non-itemizer charitable deduction and used $67,802 of taxable income instead of $65,802. It also miscalculated the bracket tax on that income ($7,610 instead of $7,640.24). The correct figure is $7,400.24 less the $2,200 CTC, or $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent child's $45,000 of wages to the joint AGI ($145,002), but the parents' unit AGI is only $100,002. That raised taxable income to $110,802 instead of $65,802. Its deductions and $2,200 CTC were otherwise correct."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It projected 2026 figures instead of using the enacted OBBBA ones: a standard deduction of about $30,750 instead of $32,200, a 10% bracket top of $24,450 instead of $24,800, and a $2,000 CTC instead of $2,200. It also omitted the $2,000 non-itemizer charitable deduction, so it reached $5,821 instead of $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It took itemized deductions of $18,557 even though they are below the $32,200 standard deduction. It also skipped the $2,000 non-itemizer charitable deduction, then submitted $15,300, which contradicts its own $7,300 estimate. Taxable income should be $65,802 and the tax after the $2,200 CTC $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It took a $13,650 qualified overtime deduction for the head, who is not flagged as paid hourly and gets none. It also added $3,220 of tax from a separate return for the child, which is not part of this output. Without those two items its method gives $65,802 of taxable income and $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in the joint AGI ($145,002 instead of $100,002). It also used a $30,800 standard deduction instead of $32,200, a $24,250 10% bracket top, and a $2,000 CTC instead of $2,200, and it omitted the $2,000 non-itemizer charitable deduction."
+us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA sunset law, with personal exemptions, 15%/25% brackets and a $1,000 CTC, even though OBBBA made the TCJA structure permanent for 2026: a $32,200 standard deduction and a $2,200 CTC. It also added a separate tax for the child, which this output excludes."
+us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It used pre-TCJA 2026 rules, with $5,300 personal exemptions, 15% brackets and a $1,000 CTC, instead of OBBBA's $32,200 standard deduction, 12% bracket and $2,200 CTC. It also added $4,896 of tax for the child's own return, which the parents' tax unit output does not include."
+us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed the TCJA had expired, applying personal exemptions, pre-TCJA brackets and a $1,000 CTC instead of OBBBA's $32,200 standard deduction plus $2,000 non-itemizer charitable deduction and $2,200 CTC. It also added a separate $4,855 tax for the child."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA sunset law (exemptions and a $1,000 CTC) and invented $16,800 of mortgage interest from the balance, although unlisted interest paid is zero. It then added $4,955 of tax from the child's separate return, which this output excludes."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It built the joint return on $145,002 of AGI, including the dependent child's $45,000 of wages that is not in the parents' AGI. It gave no tax computation behind $17,290. Correctly, $100,002 less $34,200 of deductions is $65,802, which yields $7,400.24 before the $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA 2026 rules (personal exemptions, itemizing, and a $1,000 CTC) instead of OBBBA's permanent $32,200 standard deduction and $2,200 CTC. It also added about $4,925 of tax from the child's separate return."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed TCJA expiration, with exemptions, 15% brackets and a $1,000 CTC, instead of the OBBBA 2026 parameters: a $32,200 standard deduction, a $2,000 non-itemizer charitable deduction and a $2,200 CTC. It then added $5,023 of tax for the child's own return."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. The correct computation is $100,002 AGI less the $32,200 standard deduction and the $2,000 non-itemizer charitable deduction, which leaves $65,802 taxable, $7,400.24 of tax, and $5,200.24 after the $2,200 CTC. Its $5,304 implies about $867 more taxable income, which fits approximated deduction or bracket figures rather than the exact 2026 parameters."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It used $145,002 of AGI, including the dependent child's wages, and applied pre-TCJA personal exemptions with a CTC phaseout. Under 2026 law the parents' AGI is $100,002 and the full $2,200 CTC applies well below the $400,000 phaseout threshold."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in the joint AGI ($145,002) and applied pre-TCJA itemizing with personal exemptions and no CTC. It should have used $100,002 of AGI, the $32,200 standard deduction plus the $2,000 non-itemizer charitable deduction, and the $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's $45,000 of wages into the joint AGI ($145,002). It also used the 2025 $30,000 standard deduction and 2025 brackets, a $2,000 CTC instead of $2,200, and no $2,000 non-itemizer charitable deduction."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly used $100,002 of AGI and the standard deduction. However, it applied a $2,000 CTC instead of the 2026 $2,200 and never subtracted the $2,000 non-itemizer charitable deduction. Its $5,736 is consistent with those omissions plus a sub-$32,200 standard deduction, rather than $65,802 taxable leading to $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It added $3,312 of tax from a separate return for the dependent child, which this parents' tax unit output does not include. It also used a projected $30,750 standard deduction instead of $32,200, a $2,000 CTC instead of $2,200, and no $2,000 non-itemizer charitable deduction."
+us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It put the child's wages into the joint AGI and then stacked a second $16,100 dependent standard deduction onto the joint return. It also treated the CTC as fully refundable, but the $2,200 CTC first offsets liability as a nonrefundable credit and so reduces this output."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It treated the joint return as having about $145,000 of wages, including the dependent child's $45,000 that is not part of the parents' AGI. That gave about $116,751 of taxable income instead of $65,802, and its $7,313 does not follow from that base either."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no computation. Its $14,677 fits taxing combined household wages of $145,002, including the child's earnings, without fully applying the $2,200 CTC. The correct base is $100,002 of AGI less $34,200 of deductions, which gives $65,802."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in the joint AGI ($145,002 instead of $100,002) and omitted the $2,000 non-itemizer charitable deduction. That produced $112,802 of taxable income instead of $65,802."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the dependent child's $45,000 of wages in the joint AGI ($145,002), but the parents' unit AGI is $100,002. That raised taxable income from $65,802 to $110,802. Its deductions and $2,200 CTC were otherwise correct."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002 of AGI, including the child's wages, and used the 2025 $30,000 standard deduction and 2025 brackets instead of the 2026 $32,200 and $24,800 figures. It also omitted the $2,000 non-itemizer charitable deduction."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed the parents' joint tax as $5,200.24 but then added $3,220 of tax from a separate return for the dependent child. This output is the parents' tax unit liability only."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It added $3,220 of tax from the child's own return, which this output excludes. It also omitted the $2,000 non-itemizer charitable deduction, using $67,802 of taxable income instead of $65,802."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It took a $13,650 qualified overtime deduction for the head, who is not flagged as paid hourly. It omitted the $2,000 non-itemizer charitable deduction, and it added $3,220 of separate tax for the child, which this output excludes."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It got the joint return right at $5,200.24 but added $3,220 of tax from the dependent child's separate wage return. The requested output covers the parents' tax unit only."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in the joint AGI, giving $145,002, and omitted the $2,000 non-itemizer charitable deduction. Its $12,040.44 equals tax on $112,802 less $2,200, instead of tax on $65,802 less $2,200."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It combined the head's and the child's wages into a $145,002 AGI and deducted invented mortgage interest, although unlisted interest paid is zero. Its rough $14,000 replaced the correct $65,802 taxable base and $5,200.24 result."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied TCJA-sunset rules (personal exemptions, 15% brackets and a $1,000 CTC) instead of OBBBA's $32,200 standard deduction and $2,200 CTC. It added PA income tax to itemized deductions even though no state tax paid is listed, and it added the child's separate $4,851 of tax."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It assumed TCJA expiration and phased out the CTC at a $110,000 threshold. Under OBBBA, the 2026 $2,200 CTC phases out only above $400,000. It also included the child's wages in the joint AGI ($145,002 instead of $100,002) and deducted unlisted PA income tax."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It applied pre-TCJA law (three $5,400 exemptions, 15%/25% brackets, and a CTC phaseout at $110,000) instead of the permanent OBBBA parameters. It also included the child's $45,000 in the joint AGI and itemized an unlisted PA income tax of $4,451.56."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It used pre-TCJA 2026 rules: itemizing with three $5,434 exemptions, a 10%/15% bracket structure and a $1,000 CTC, instead of the $32,200 standard deduction, the $2,000 non-itemizer charitable deduction and the $2,200 CTC. It then added $4,846 of tax from the child's own return."
+us,scenario_123,federal_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It reached the correct $65,802 of taxable income for the joint return but then added about $3,225 of tax from a separate return for the dependent child. This output is the parents' tax unit only, which is $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no value and no explanation for federal_income_tax_before_refundable_credits.
+us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in the joint AGI ($145,002) and took a $12,000 qualified overtime deduction for the head, who is not flagged as paid hourly. It also left out the $2,000 non-itemizer charitable deduction."
+us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It claimed that credits wipe out the liability, but the only nonrefundable credit is the $2,200 CTC, set against $7,400.24 of tax on $65,802 of taxable income. That leaves $5,200.24. Refundable credits do not reduce this output at all."
+us,scenario_123,federal_income_tax_before_refundable_credits,muse-spark-1.3,prompt_ambiguity,household_unit_or_filing_status,False,"It omitted the 2026 $2,000 non-itemizer charitable deduction for joint filers and used $67,802 of taxable income instead of $65,802. It also slightly misstated the bracket tax ($7,644.24 instead of $7,640.24)."
+us,scenario_123,federal_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"It used a joint AGI of $145,002 that includes the dependent child's wages. It also took a $13,650 overtime deduction for the head, who is not flagged as paid hourly, and used a $31,500 standard deduction instead of the 2026 $32,200."
+us,scenario_123,federal_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in the joint AGI and applied a $12,500 overtime deduction for the head, who is not flagged as paid hourly. It also omitted the $2,000 non-itemizer charitable deduction, so it computed tax on $100,302 instead of $65,802."
+us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It imputed $27,300 of mortgage interest from the $420,000 balance, although no interest paid is listed and the amount is zero, and so itemized instead of taking the $32,200 standard deduction. It also included the child's wages in AGI, used 2025 brackets, and applied a $2,000 CTC instead of $2,200."
+us,scenario_123,federal_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It made up a QBI deduction on W-2 wages, $16,800 of mortgage interest, a CDCC with no care expenses and a foreign earned income exclusion, then zeroed out a liability it had itself computed as $7,687. The only nonrefundable credit is the $2,200 CTC against $7,400.24 of tax, which leaves $5,200.24."
us,scenario_123,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model paid out a flat $1,600 ""refundable CTC"" without running the ACTC limitation: the Child Tax Credit is first applied as a nonrefundable credit, and with MFJ taxable income of roughly $112,800 (AGI $145,002 less the 2026 MFJ standard deduction) generating about $14,000 of federal income tax, the entire CTC is absorbed nonrefundably and no additional child tax credit arises. It compounded this by inventing $205 of EITC at $145,002 of AGI, when the one-child EITC for joint filers is fully phased out in the upper $50,000s."
us,scenario_123,federal_refundable_credits,glm-5.3,llm_error,credit_phaseout,False,"The model asserted that satisfying the 15%-of-earnings phase-in makes ""the full credit refundable,"" skipping the step that governs here: the CTC offsets the household's roughly $14,000 federal income tax liability as a nonrefundable credit first, and only the unused remainder — zero — flows to the refundable additional child tax credit, which is separately capped at $1,700 per child in 2026 rather than the full $2,400 the model reported."
us,scenario_123,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model claimed ""no limitation from the nonrefundable portion,"" which is exactly backwards: the nonrefundable CTC is applied first against the household's roughly $14,000 of federal income tax before refundable credits, exhausting the credit and leaving $0 of additional child tax credit. Its $1,500 figure is also a stale pre-2022 ACTC cap rather than the 2026 $1,700 ceiling."
@@ -10843,57 +10962,58 @@ us,scenario_123,federal_refundable_credits,inkling,llm_error,credit_phaseout,Fal
us,scenario_123,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value and no explanation for federal_refundable_credits, so the submission failed the output contract rather than expressing a substantive tax error. The correct derivation absorbs the full 2026 CTC against roughly $14,000 of pre-credit federal income tax, leaving $0 refundable."
us,scenario_123,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model checked the two limits that do not bind — the $2,500 earned-income floor and the $400,000 MFJ phase-out — and never applied the one that does, namely that the ACTC is only the CTC remaining after the nonrefundable credit offsets the household's roughly $14,000 federal income tax before refundable credits. Its EITC conclusion of $0 was right; the $1,700 ACTC is the entire error."
us,scenario_123,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model reasoned that the child's own $45,000 of W-2 wages makes the credit ""fully refundable up to the 2026 CTC cap of $2,000,"" conflating two distinct limits: earned income only sets the 15% phase-in ceiling on the ACTC, the 2026 CTC is $2,200 with a $1,700 refundable cap, and the credit is exhausted nonrefundably against the household's roughly $14,000 of federal income tax, leaving nothing refundable."
-us,scenario_123,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"Correctly found household income of $145,002 and no SNAP/TANF categorical eligibility, but treated the federal 130% FPG test as the only way to get free meals. Never applied Pennsylvania's universal free breakfast (Act 33 of 2023), which gives the 16-year-old free breakfast worth $370.80/year at any income."
-us,scenario_123,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"Stopped at the federal 130% FPL and SNAP/TANF categorical tests and ignored Pennsylvania's universal free breakfast under 24 P.S. § 13-1337.1(c.2). That benefit gives positive free-meal support ($2.06/day × 180 days = $370.80) regardless of the $145,002 income."
-us,scenario_123,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"Applied only the federal 130% FPL lunch test, using the wrong household size (4 instead of 3), and wrongly brought in assets and a mortgage balance, which play no role in school meal eligibility. Missed Pennsylvania's income-blind universal free breakfast, which makes the student's breakfast free and the support positive."
-us,scenario_123,free_school_meals_eligible,claude-haiku-5.5,llm_error,state_local_rule,False,"Concluded that high income rules out free meals, which only holds for the federal income-tested tier. Pennsylvania's Act 33 universal free breakfast gives the 16-year-old student $370.80/year of free breakfast whatever the household income."
-us,scenario_123,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"Relied only on the federal 130% FPL free-meal threshold and never accounted for Pennsylvania's universal free school breakfast. That program makes breakfast free for the K-12 student at any income, so free-meal support is positive."
-us,scenario_123,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,Treated the federal 130% poverty-guideline test as the only route to free meals and ignored Pennsylvania's Act 33 universal free breakfast. That program puts breakfast in the free tier worth $370.80/year even though lunch stays paid.
-us,scenario_123,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,Compared income only to the federal 130% FPL free-meal limit and missed Pennsylvania's universal free breakfast for K-12 students. That breakfast produces positive free-meal support of $370.80 regardless of income.
-us,scenario_123,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,Applied only the federal 130% FPL free-meal limit and never applied Pennsylvania's income-blind universal free breakfast (24 P.S. § 13-1337.1(c.2)). That benefit makes the 16-year-old's breakfast free and the support positive.
-us,scenario_123,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"Correctly summed $145,002 and found it far above 130% FPL, but treated the NSLP income test as decisive. Missed Pennsylvania's universal free breakfast, which gives free breakfast ($2.06/day × 180 = $370.80) to the student at any income."
-us,scenario_123,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"Applied the federal 185% reduced-price and 130% free-meal income limits and ignored Pennsylvania's Act 33 universal free breakfast. That program makes breakfast free for the 16-year-old regardless of the $145,002 income, so free-meal support is positive."
-us,scenario_123,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"Relied solely on the 130% FPL federal free-meal test and missed Pennsylvania's universal free breakfast, which gives the K-12 student $370.80/year of free breakfast at any income."
-us,scenario_123,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,Used only the federal 130% FPL free-meal income limit and never applied Pennsylvania's income-blind universal free breakfast. That breakfast makes the household's free-meal support positive.
-us,scenario_123,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"Compared $145,002 to a 130% FPL limit and stopped there, missing Pennsylvania's Act 33 universal free breakfast. That program gives the student free breakfast worth $370.80/year regardless of income."
-us,scenario_123,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,Treated the federal 130% poverty-guideline threshold as the only free-meal pathway and ignored Pennsylvania's universal free school breakfast. That breakfast places the student's breakfast in the free tier at any income.
-us,scenario_123,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"Applied only the 130% FPL federal free-meal limit and missed Pennsylvania's income-blind universal free breakfast, which makes free-meal support positive ($370.80/year)."
-us,scenario_123,free_school_meals_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"Correctly found $145,002 above 130% FPL but treated that federal test as decisive. Missed Pennsylvania's universal free breakfast under 24 P.S. § 13-1337.1(c.2), which makes the student's breakfast free at any income."
-us,scenario_123,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,Applied only an income threshold for free meals and ignored Pennsylvania's universal free breakfast for K-12 students. That program produces positive free-meal support regardless of household income.
-us,scenario_123,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"Relied on the federal 130% FPL free-meal limit alone and never applied Pennsylvania's Act 33 universal free breakfast. That program makes breakfast free for the 16-year-old, worth $370.80 for the year."
-us,scenario_123,free_school_meals_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,"Concluded that Child 1 is ineligible because income exceeds 130% FPL, but missed that Pennsylvania's universal free breakfast gives Child 1 free breakfast regardless of income. That yields positive free-meal support."
-us,scenario_123,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"Applied only the federal income threshold and ignored Pennsylvania's income-blind universal free breakfast, which gives the K-12 student $2.06/day × 180 days = $370.80 of free-meal support."
-us,scenario_123,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,Treated the 130% FPL federal free-meal test as the only pathway and missed Pennsylvania's universal free school breakfast. That program puts breakfast in the free tier for the student at any income.
-us,scenario_123,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,Stopped at the federal 130% poverty-guideline threshold and never applied Pennsylvania's Act 33 universal free breakfast. That program makes the household's free-meal support positive.
-us,scenario_123,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,Applied only the federal 130% FPG income test and ignored Pennsylvania's universal free breakfast for K-12 students. That breakfast is free regardless of income and worth $370.80/year here.
-us,scenario_123,free_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,"Correctly placed $145,002 well above 130% FPL for three but treated that as conclusive. Missed Pennsylvania's universal free breakfast (24 P.S. § 13-1337.1(c.2)), which makes the 16-year-old's breakfast free and the support positive."
-us,scenario_123,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,"Returned no value or explanation for free_school_meals_eligible, so there was no answer to score. Getting it right required applying Pennsylvania's universal free breakfast, which gives positive free-meal support and a value of 1."
-us,scenario_123,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,Compared income only to the 130% FPL free-meal limit for three and missed Pennsylvania's income-blind universal free breakfast. That program gives the student $370.80/year of free breakfast.
-us,scenario_123,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,Assumed that income-based thresholds are the only source of free-meal support and ignored Pennsylvania's universal free breakfast. Under that program the K-12 student gets positive free breakfast support at any income.
-us,scenario_123,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"Required means-tested or categorical indicators for free meals and missed Pennsylvania's universal free breakfast, which needs neither. It makes the 16-year-old's breakfast free and worth $370.80/year."
-us,scenario_123,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"Correctly found income above 130% FPG with no categorical eligibility, but treated the federal tiers as exhaustive. Missed Pennsylvania's Act 33 universal free breakfast, which makes breakfast free regardless of income."
-us,scenario_123,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,Applied only an income threshold and never accounted for Pennsylvania's universal free school breakfast. That program produces positive free-meal support ($370.80) for the K-12 student regardless of income.
-us,scenario_123,free_school_meals_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,Considered only the federal income and categorical pathways and missed Pennsylvania's income-blind universal free breakfast. That breakfast places the student's breakfast in the free tier and makes support positive.
-us,scenario_123,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"Relied solely on the federal free-meal income threshold and ignored Pennsylvania's Act 33 universal free breakfast, which gives free breakfast worth $2.06/day over 180 school days."
-us,scenario_123,free_school_meals_eligible,gpt-6-astra,llm_error,state_local_rule,False,Checked only the federal income limit and categorical eligibility and missed Pennsylvania's universal free breakfast under 24 P.S. § 13-1337.1(c.2). That program makes free-meal support positive regardless of income.
-us,scenario_123,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,Applied the federal free-meal income guideline for three as the only test and never applied Pennsylvania's universal free breakfast. That program gives the 16-year-old free breakfast at any income.
-us,scenario_123,free_school_meals_eligible,gpt-6-sol,llm_error,state_local_rule,False,"Limited the analysis to the federal income threshold and categorical eligibility and missed Pennsylvania's income-blind universal free breakfast, which yields $370.80 of free-meal support."
-us,scenario_123,free_school_meals_eligible,gpt-6.1-sol,llm_error,state_local_rule,False,"Correctly ruled out income-based federal free meals at >130% FPG, but treated that as the full answer and missed Pennsylvania's Act 33 universal free breakfast. That program makes breakfast free for the K-12 student."
-us,scenario_123,free_school_meals_eligible,gpt-6.1-terra,llm_error,state_local_rule,False,"Applied only the federal free-meal income limit and ignored Pennsylvania's universal free school breakfast, which gives free breakfast regardless of household income and makes support positive."
-us,scenario_123,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,Referred to Pennsylvania's income eligibility but missed that Pennsylvania gives universal free breakfast under Act 33 of 2023 with no income test. That makes the student's breakfast free and worth $370.80/year.
-us,scenario_123,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"Relied on the 130% FPL limit and the absence of categorical eligibility, and never applied Pennsylvania's income-blind universal free breakfast, which places breakfast in the free tier."
-us,scenario_123,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,"Asserted that PolicyEngine's free-meal support is zero based only on the federal 130% FPG and categorical tests. PolicyEngine also applies Pennsylvania's universal free breakfast, valued at $2.06/day × 180 days = $370.80, so support is positive."
-us,scenario_123,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"Applied only the federal 130% FPL and SNAP/TANF categorical pathways and missed Pennsylvania's Act 33 universal free breakfast, which makes the 16-year-old's breakfast free regardless of income."
-us,scenario_123,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"Compared $145,002 to a ~$34,200 130% FPL limit and stopped there, missing Pennsylvania's universal free breakfast. That program gives free breakfast to the K-12 student at any income."
-us,scenario_123,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,"Treated the 130% FPL federal test as the only free-meal pathway and ignored Pennsylvania's income-blind universal free breakfast, which produces $370.80 of free-meal support."
-us,scenario_123,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"Applied only the 130% FPL federal test, and wrongly treated the 16-year-old as not in K-12 even though PolicyEngine treats a school-age child as a K-12 student. Missed Pennsylvania's universal free breakfast, which makes that student's breakfast free regardless of income."
-us,scenario_123,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"Considered only SNAP/TANF categorical eligibility and the 130% FPL limit, and missed Pennsylvania's Act 33 universal free breakfast. That program gives positive free-meal support with no income test."
-us,scenario_123,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,Assumed Pennsylvania free meals are income-tested only and missed the state's universal free school breakfast under 24 P.S. § 13-1337.1(c.2). That program makes the student's breakfast free at any income.
-us,scenario_123,free_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,"Applied only the 130% FPL threshold and never applied Pennsylvania's universal free breakfast, which gives the K-12 student $370.80/year of free breakfast regardless of income."
-us,scenario_123,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,Treated the federal 130% FPL free-meal limit as decisive and missed Pennsylvania's income-blind universal free breakfast. That program makes free-meal support positive.
-us,scenario_123,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"Estimated income at ~544% of poverty but treated the 130% test as the only free-meal pathway. Missed Pennsylvania's Act 33 universal free breakfast, which gives the student free breakfast regardless of income."
-us,scenario_123,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"Applied only the federal 130% FPL free-meal limit and ignored Pennsylvania's universal free school breakfast. That program makes the 16-year-old's breakfast free, worth $2.06/day over 180 days."
-us,scenario_123,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,"Relied solely on an income limit and missed Pennsylvania's income-blind universal free breakfast for K-12 students, which yields positive free-meal support of $370.80."
+us,scenario_123,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,"The model correctly found income at about 545% of FPL with no SNAP/TANF categorical eligibility, but it treated the federal 130% test as the only route to free meals. It missed Pennsylvania's universal free breakfast under Act 33 of 2023, which makes the 16-year-old's breakfast free ($370.80 a year) regardless of income."
+us,scenario_123,free_school_meals_eligible,claude-fable-5.1,llm_error,state_local_rule,False,"The model concluded there was no free meal support because income exceeds 130% FPL and there is no categorical eligibility. It ignored Pennsylvania's income-blind universal school breakfast, which produces $370.80 of positive free-meal support for the K-12 child."
+us,scenario_123,free_school_meals_eligible,claude-fable-5.2,llm_error,state_local_rule,False,"The model relied only on income being far above 130% of poverty. It never applied Pennsylvania's Act 33 universal free breakfast, which makes breakfast free for the 16-year-old student whatever the household income."
+us,scenario_123,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,"The model used a family-of-4 poverty line for a 3-person household and wrongly cited assets, even though school meals have no asset test. Most importantly, it missed Pennsylvania's universal free breakfast, which gives positive free-meal support ($370.80) regardless of income."
+us,scenario_123,free_school_meals_eligible,claude-haiku-5.5,llm_error,state_local_rule,False,"The model applied only the income limit for free meals. It missed that Pennsylvania serves a free breakfast to all students at participating schools regardless of income, so the 16-year-old receives $370.80 of free breakfast support."
+us,scenario_123,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,"The model treated the federal 130% FPL threshold as decisive. It overlooked Pennsylvania's Act 33 universal free breakfast, which puts the K-12 child's breakfast in the free tier at any income."
+us,scenario_123,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model concluded that income above 130% FPL means no free school meal support. Pennsylvania's universal free breakfast still provides $2.06 a day over 180 school days ($370.80) for the 16-year-old, so the support is positive."
+us,scenario_123,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model applied only the 130% FPL free-meal income test. It missed Pennsylvania's income-blind free school breakfast (24 P.S. § 13-1337.1(c.2)), which makes the free-meal value positive."
+us,scenario_123,free_school_meals_eligible,claude-opus-5.5,llm_error,state_local_rule,False,"The model stopped at the federal 130% FPL limit. It ignored that Pennsylvania provides a universal free breakfast to K-12 students, which gives this household $370.80 of free-meal support."
+us,scenario_123,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model computed $145,002 of income against a 130% FPL limit for a family of 3 and stopped there. It never considered Pennsylvania's Act 33 universal free breakfast, which makes the 16-year-old's breakfast free regardless of income."
+us,scenario_123,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,"The model correctly placed income above even the 185% reduced-price limit. It then wrongly concluded that no free meals apply, missing Pennsylvania's universal free breakfast, which provides $370.80 of free-tier breakfast support for the student."
+us,scenario_123,free_school_meals_eligible,claude-sonnet-5.5,llm_error,state_local_rule,False,"The model judged eligibility only against 130% FPL. It missed Pennsylvania's state-funded free breakfast for all students at participating schools, which yields positive free-meal support here."
+us,scenario_123,free_school_meals_eligible,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"The model applied only the federal 130% FPL free-meal limit. It missed Pennsylvania's universal free breakfast, which makes the K-12 child's breakfast free regardless of household income."
+us,scenario_123,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model compared $145,002 with a roughly $35,750 limit (130% FPL) and returned 0. It ignored Pennsylvania's Act 33 universal free breakfast, which provides $370.80 of free breakfast support for the 16-year-old."
+us,scenario_123,free_school_meals_eligible,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"The model relied solely on the 130% poverty-guideline test for free meals. It missed that Pennsylvania offers a free breakfast to every student at participating schools, so the free-meal support is positive."
+us,scenario_123,free_school_meals_eligible,deepseek-v4.1-flash,llm_error,state_local_rule,False,"The model applied only the 130% FPL limit. It did not account for Pennsylvania's universal free breakfast, which puts the student's breakfast in the free tier at any income."
+us,scenario_123,free_school_meals_eligible,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly found $145,002 above 130% FPL but treated that as the end of the analysis. It missed Pennsylvania's income-blind universal free breakfast, which makes the free-meal value positive."
+us,scenario_123,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model applied only an income threshold. It ignored Pennsylvania's Act 33 universal free breakfast, which gives the 16-year-old K-12 student free breakfast regardless of income."
+us,scenario_123,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model concluded the household was ineligible because income exceeds 130% FPL. It missed Pennsylvania's universal free school breakfast, which provides $370.80 of free-meal support."
+us,scenario_123,free_school_meals_eligible,gemini-3.5-flash,llm_error,state_local_rule,False,"The model ruled Child 1 out because income exceeds 130% FPL. Under Pennsylvania's universal breakfast law, the child receives a free breakfast regardless of household income, so free-meal support is positive."
+us,scenario_123,free_school_meals_eligible,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"The model applied only the income threshold. It missed Pennsylvania's free breakfast for all students at participating schools, which yields $370.80 of free-meal support."
+us,scenario_123,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,"The model treated the 130% FPL test as the only pathway. It missed Pennsylvania's Act 33 universal free breakfast, which places the student's breakfast in the free tier."
+us,scenario_123,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"The model applied only the 130% poverty-guideline threshold. It ignored that Pennsylvania serves a free breakfast to K-12 students regardless of income, so free-meal support is positive."
+us,scenario_123,free_school_meals_eligible,gemini-3.8-flash,llm_error,state_local_rule,False,"The model concluded the household was ineligible based only on the 130% FPL limit. It missed Pennsylvania's universal free breakfast, which gives the 16-year-old $370.80 of free breakfast."
+us,scenario_123,free_school_meals_eligible,gemini-4-argon,llm_error,state_local_rule,False,"The model correctly placed $145,002 well above 130% FPL for three people. It then missed Pennsylvania's income-blind universal school breakfast, which still yields positive free school meal support."
+us,scenario_123,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model returned no usable value or explanation for free_school_meals_eligible. The correct answer is 1 because Pennsylvania's universal free breakfast gives the 16-year-old student $370.80 of free-meal support.
+us,scenario_123,free_school_meals_eligible,glm-5.3,llm_error,state_local_rule,False,"The model applied only the 130% of poverty free-meal limit for a family of 3. It missed Pennsylvania's Act 33 universal free breakfast, which makes breakfast free for the K-12 child regardless of income."
+us,scenario_123,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,The model assumed income alone determines whether positive free-meal support exists. Pennsylvania's universal free breakfast produces $370.80 of free support at any income level.
+us,scenario_123,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,The model returned 0 because of high wages and no means-tested indicators. It missed that Pennsylvania's universal school breakfast is not means-tested and gives the 16-year-old a free breakfast.
+us,scenario_123,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model correctly found no categorical eligibility and income well above 130% FPL. It missed Pennsylvania's universal free breakfast, which places the student's breakfast in the free tier regardless of income."
+us,scenario_123,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,"The model applied only an income threshold for positive free-meal support. It ignored Pennsylvania's Act 33 free breakfast for all students at participating schools, which yields $370.80."
+us,scenario_123,free_school_meals_eligible,gpt-5.6-sol,llm_error,state_local_rule,False,"The model concluded there was no positive free-meal support because of income and the lack of categorical eligibility. It missed Pennsylvania's universal free breakfast, which provides free-tier breakfast support with no income test."
+us,scenario_123,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model relied solely on the free-meal income threshold. It overlooked Pennsylvania's income-blind universal school breakfast, which makes free-meal support positive."
+us,scenario_123,free_school_meals_eligible,gpt-6-astra,llm_error,state_local_rule,False,The model checked only the income limit and categorical eligibility. Pennsylvania's Act 33 universal free breakfast is a third pathway that gives the K-12 student $370.80 of free breakfast.
+us,scenario_123,free_school_meals_eligible,gpt-6-luna,llm_error,state_local_rule,False,"The model applied the federal free-meal income guideline for three people and stopped. It missed Pennsylvania's universal free breakfast, which provides free meals to the 16-year-old regardless of income."
+us,scenario_123,free_school_meals_eligible,gpt-6-sol,llm_error,state_local_rule,False,"The model treated the income threshold and categorical eligibility as the only routes to free meals. It missed Pennsylvania's universal free breakfast, which yields positive free-meal support."
+us,scenario_123,free_school_meals_eligible,gpt-6.1-sol,llm_error,state_local_rule,False,"The model explicitly checked only income-based free meals against 130% of poverty. It never applied Pennsylvania's non-income-based universal free breakfast, which makes the support positive."
+us,scenario_123,free_school_meals_eligible,gpt-6.1-terra,llm_error,state_local_rule,False,"The model applied only the free-meal income limit. It missed that Pennsylvania gives a free breakfast to all students at participating schools, producing $370.80 of free-meal support."
+us,scenario_123,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,"The model cited Pennsylvania but applied only the income-eligibility test. It missed Pennsylvania's own Act 33 universal free breakfast, which makes the student's breakfast free regardless of income."
+us,scenario_123,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,"The model checked the 130% FPL limit and categorical eligibility only. It missed Pennsylvania's universal free breakfast, which puts breakfast in the free tier for the 16-year-old."
+us,scenario_123,free_school_meals_eligible,grok-4.6,llm_error,state_local_rule,False,The model concluded free-meal support was zero after ruling out the 130% FPL and categorical routes. PolicyEngine applies Pennsylvania's universal free breakfast and returns $370.80 of free breakfast support.
+us,scenario_123,free_school_meals_eligible,grok-4.7,llm_error,state_local_rule,False,"The model applied only the income and SNAP/TANF categorical tests. It missed Pennsylvania's income-blind universal school breakfast under Act 33, which gives positive free-meal support."
+us,scenario_123,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model compared $145,002 with a roughly $34,200 limit (130% FPL) and returned 0. It missed Pennsylvania's universal free breakfast, which provides free breakfast to the K-12 student at any income."
+us,scenario_123,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,The model assumed income above 130% FPL means no free-meal support. Pennsylvania's universal free breakfast still yields $370.80 of free support for the 16-year-old.
+us,scenario_123,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"The model applied only the 130% FPL test and wrongly treated the 16-year-old as not enrolled in school. PolicyEngine treats the child as a K-12 student, and Pennsylvania's universal free breakfast applies to that student regardless of income."
+us,scenario_123,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model ruled out free meals using only categorical eligibility and the 130% FPL limit. It missed Pennsylvania's Act 33 universal free breakfast, which provides free-tier breakfast regardless of income."
+us,scenario_123,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,"The model applied an income threshold even though it named Pennsylvania. It missed that Pennsylvania provides a free breakfast to all students at participating schools, which makes free-meal support positive."
+us,scenario_123,free_school_meals_eligible,muse-spark-1.3,llm_error,state_local_rule,False,"The model relied only on the 130% FPL limit. It ignored Pennsylvania's universal free breakfast, which gives the student $370.80 of free breakfast."
+us,scenario_123,free_school_meals_eligible,muse-wonder,llm_error,state_local_rule,False,"The model treated the 130% FPL limit as the only route to free meals. It missed Pennsylvania's income-blind universal school breakfast, which puts breakfast in the free tier."
+us,scenario_123,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,"The model correctly placed income at about 544% of poverty but applied only the federal 130% rule. It missed Pennsylvania's Act 33 universal free breakfast, which provides free-meal support regardless of income."
+us,scenario_123,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"The model checked only the federal 130% FPL free-meal limit for the 16-year-old. It missed Pennsylvania's universal free breakfast, which makes that child's breakfast free and the support positive."
+us,scenario_123,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,"The model applied only the free-meal income limit. It overlooked Pennsylvania's universal free breakfast for K-12 students, which yields $370.80 of free-meal support regardless of income."
us,scenario_123,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable head_medicaid_eligible value or explanation, violating the required function-output contract."
us,scenario_123,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly asserted that the household's MAGI was below Pennsylvania's adult Medicaid threshold, even though the trace places MAGI at 5.31 times FPL. It also treated being under age 65 as sufficient categorical status, while the head qualifies through no Medicaid pathway."
@@ -10902,99 +11022,101 @@ us,scenario_123,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,
us,scenario_123,local_income_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model assumed the household lives in Philadelphia and applied the 3.8712% resident wage tax to the Head's and Child 1's wages, but the facts list only PA with no locality. A PA household with no locality is outside every jurisdiction this output covers, so the liability is $0. Its final $3,545 also does not match its own figures ($3,871.20 + $1,742.04): it lowered the total by an arbitrary amount for a 'limited exemption' it never identified."
us,scenario_123,local_income_tax,claude-haiku-5.5,llm_error,state_local_rule,False,"The model applied a generic Pennsylvania municipal earned income tax of about 3.67% to $145,000 of combined wages. The output is limited to NYC, Philadelphia, Kansas City and St. Louis taxes, and no locality is listed for this household. Ordinary PA municipal and school-district EITs are not part of local_income_tax, so the correct value is $0."
us,scenario_123,local_income_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model said Philadelphia is 'the relevant PA locality' just because the household is in PA, and applied a resident wage tax of about 3.75% to $145,000 of wages. The household has no Philadelphia residence, so no covered local tax applies and the correct value is $0. Its submitted $3,830.90 also contradicts its own stated total of $5,437.50."
-us,scenario_123,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning correctly added Social Security, Medicare, and the 0.07% PA employee UC contribution ($101.50) to reach $11,194. It then submitted 11,663.35, a number that no step of its own work supports."
-us,scenario_123,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed only federal FICA (7.65% × $145,000 = $11,092.50). It left out Pennsylvania's mandatory employee Unemployment Compensation contribution of 0.07% on all wages, which is $101.50."
-us,scenario_123,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It stated that the head's $100,000 is below the Additional Medicare Tax threshold, then added $648.89 of Additional Medicare Tax anyway. It also wrongly said Pennsylvania has no mandatory employee payroll tax, which dropped the 0.07% UC contribution of $101.50."
-us,scenario_123,payroll_tax,claude-haiku-5.5,llm_error,other,False,"With every wage under the Social Security cap, its stated method (6.2% + 1.45% on $145,000) gives $11,092.50, but it reported $13,500, an arithmetic error. It also left out Pennsylvania's 0.07% employee UC contribution ($101.50), which brings the correct total to $11,194."
-us,scenario_123,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It correctly computed the PA employee UC contribution as 0.07% × $145,000 = $101.50, then set it to $0 in the total by wrongly calling it not mandatory. Its answer was $11,092.50 instead of $11,194."
-us,scenario_123,payroll_tax,claude-opus-4.8,llm_error,other,False,"Its reasoning correctly summed Social Security, Medicare, and the 0.07% PA UC contribution to $11,194. It then submitted 11,097.50, which contradicts its own correct calculation."
-us,scenario_123,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,It applied only the 7.65% federal FICA rate. It never added Pennsylvania's mandatory employee UC contribution of 0.07% on all wages ($101.50).
-us,scenario_123,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It took 7.65% of $145,000 and stopped there. It left out Pennsylvania's mandatory 0.07% employee Unemployment Compensation contribution ($101.50), which the question's definition of payroll tax includes."
-us,scenario_123,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It recognized the PA employee UC contribution but wrongly capped it at the first $10,000 of each worker's wages, giving $14. Pennsylvania withholds the 0.07% employee contribution on all gross wages with no cap, which is $101.50."
-us,scenario_123,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no UI withholding from employees. Pennsylvania withholds 0.07% of all wages for Unemployment Compensation, which is $101.50 here, so the total is $11,194."
-us,scenario_123,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It wrongly assumed there is no employee state payroll tax. It left out Pennsylvania's mandatory 0.07% employee UC contribution on the $145,000 in wages ($101.50)."
-us,scenario_123,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,It computed only federal Social Security and Medicare. It left out Pennsylvania's mandatory 0.07% employee Unemployment Compensation contribution ($101.50).
-us,scenario_123,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It added up per-person FICA to $11,092.50 and never included Pennsylvania's 0.07% employee UC withholding on all wages ($70 for the head + $31.50 for the child = $101.50)."
-us,scenario_123,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It wrongly said no mandatory state payroll tax applies. Pennsylvania withholds a 0.07% employee Unemployment Compensation contribution on all wages, which adds $101.50."
-us,scenario_123,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It computed only 6.2% Social Security and 1.45% Medicare on $145,000. It left out Pennsylvania's mandatory 0.07% employee UC contribution ($101.50)."
-us,scenario_123,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"Its stated method, 7.65% of $145,000, gives $11,092.50, but it reported $11,108, an arithmetic slip. It also left out Pennsylvania's 0.07% employee UC contribution ($101.50)."
-us,scenario_123,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It counted only Social Security and Medicare, which come to $11,092.50, and it reported an unsupported $11,138. It left out Pennsylvania's mandatory 0.07% employee UC contribution ($101.50), which brings the total to $11,194."
-us,scenario_123,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,It limited payroll tax to federal FICA. It never added Pennsylvania's 0.07% employee Unemployment Compensation withholding on the head's and child's wages ($101.50).
-us,scenario_123,payroll_tax,gemini-4-argon,llm_error,state_local_rule,False,"It took 7.65% of the $145,000 in wages. It left out Pennsylvania's mandatory 0.07% employee UC contribution ($101.50), which the question explicitly counts as a state payroll tax."
-us,scenario_123,payroll_tax,glm-5.2,llm_error,state_local_rule,False,It wrongly stated that Pennsylvania has no mandatory employee-side payroll tax. Pennsylvania's 0.07% employee UC contribution on all wages adds $101.50.
-us,scenario_123,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no mandatory employee payroll tax. That dropped the 0.07% UC contribution on $145,000 in wages ($101.50)."
-us,scenario_123,payroll_tax,gpt-5.4-mini,llm_error,other,False,"Its stated method (Social Security and Medicare on wages below the cap) gives $11,092.50, but it reported $12,366, an arithmetic error. It also wrongly assumed there is no Pennsylvania employee payroll tax, which dropped the 0.07% UC contribution ($101.50)."
-us,scenario_123,payroll_tax,gpt-5.4-nano,llm_error,other,False,"Its stated method of 6.2% plus 1.45% on $145,000 in wages gives $11,092.50, but it reported $15,399, which no part of its reasoning supports. It also left out Pennsylvania's 0.07% employee UC contribution ($101.50)."
-us,scenario_123,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,It applied only 7.65% FICA to each earner. It left out Pennsylvania's mandatory 0.07% employee Unemployment Compensation contribution ($101.50).
-us,scenario_123,payroll_tax,gpt-6-astra,llm_error,thresholds_rates,False,"It included the Pennsylvania employee UC contribution but at 0.06% ($87). The employee rate is 0.07%, which gives $101.50 on $145,000 and a total of $11,194."
-us,scenario_123,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,It wrongly said no Pennsylvania employee payroll tax applies. It left out the mandatory 0.07% UC contribution on all wages ($101.50).
-us,scenario_123,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Its stated method of 7.65% FICA gives $11,092.50, but it reported $11,107.50, which includes an unexplained extra $15. It also left out Pennsylvania's 0.07% employee UC contribution ($101.50)."
-us,scenario_123,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It wrongly said there is no mandatory PA employee payroll tax. Pennsylvania withholds 0.07% of all wages for Unemployment Compensation, which adds $101.50."
-us,scenario_123,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no mandatory employee-side payroll tax. That dropped the 0.07% employee UC contribution on $145,000 ($101.50)."
-us,scenario_123,payroll_tax,grok-4.7,llm_error,state_local_rule,False,It wrongly said Pennsylvania has no mandatory employee state payroll tax. It left out the 0.07% employee UC withholding: $70 on the head's wages and $31.50 on the child's.
-us,scenario_123,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,It wrongly concluded there is no mandatory state employee payroll tax. It left out Pennsylvania's 0.07% employee Unemployment Compensation contribution ($101.50).
-us,scenario_123,payroll_tax,inkling,llm_error,state_local_rule,False,"It wrongly set state mandatory employee payroll taxes to zero. Pennsylvania's 0.07% employee UC contribution on all wages adds $101.50, for a total of $11,194."
-us,scenario_123,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It submitted no value and no explanation for payroll_tax, so there was nothing to score against the $11,194 reference."
-us,scenario_123,payroll_tax,minimax-m3,llm_error,other,False,"It acknowledged that $145,000 in wages creates Social Security and Medicare tax, then submitted $0 without basis. The correct total is $8,990 Social Security + $2,102.50 Medicare + $101.50 PA employee UC = $11,194."
-us,scenario_123,payroll_tax,muse-spark-1.3,llm_error,state_local_rule,False,It computed only federal FICA at 7.65%. It left out Pennsylvania's mandatory 0.07% employee Unemployment Compensation contribution ($101.50).
-us,scenario_123,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"It wrongly stated that Pennsylvania levies no mandatory employee payroll tax. That dropped the 0.07% employee UC contribution on $145,000 in wages ($101.50)."
-us,scenario_123,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no state payroll taxes. It left out the mandatory 0.07% employee Unemployment Compensation contribution ($101.50) on top of the $11,092.50 in FICA."
-us,scenario_123,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It put the head's Social Security at $7,650 instead of $6,200 and invented $270 of Additional Medicare Tax, even though the couple's wages are far below the $250,000 joint threshold. It then adjusted its total arbitrarily and never applied Pennsylvania's 0.07% employee UC contribution ($101.50)."
+us,scenario_123,payroll_tax,claude-fable-5,llm_error,other,False,"Its reasoning got every component right ($6,200 + $1,450 + $2,790 + $652.50 + $70 + $31.50 PA UC) and reached $11,194. It then submitted the unrelated figure $11,663.35, so the final value contradicts its own correct working."
+us,scenario_123,payroll_tax,claude-fable-5.1,llm_error,state_local_rule,False,"It computed only the 7.65% federal FICA ($11,092.50). It left out Pennsylvania's mandatory 0.07% employee unemployment compensation contribution on all wages, which adds $101.50."
+us,scenario_123,payroll_tax,claude-fable-5.2,llm_error,state_local_rule,False,"It said there was 'no modeled PA employee payroll tax' and so dropped Pennsylvania's 0.07% employee UC withholding on the full $145,000. That missing $101.50 is what was needed to reach $11,194."
+us,scenario_123,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"It correctly said no wages exceed the Additional Medicare Tax threshold, then added a made-up $648.89 Additional Medicare Tax anyway. It also wrongly claimed Pennsylvania has no mandatory employee payroll tax, missing the 0.07% UC contribution ($101.50)."
+us,scenario_123,payroll_tax,claude-haiku-5.5,llm_error,other,False,"Its stated method (6.2% + 1.45% on $145,000 with no Additional Medicare Tax) gives $11,092.50, but it submitted $13,500, which nothing in its working supports. It also left out the $101.50 Pennsylvania employee UC contribution (0.07% of wages)."
+us,scenario_123,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"It computed the Pennsylvania employee UC tax correctly (0.07% × $145,000 = $101.50), then dropped it as 'minimal'. Because the question explicitly includes mandatory employee state payroll taxes, the right total is $11,194."
+us,scenario_123,payroll_tax,claude-opus-4.8,llm_error,other,False,"It correctly added SS, Medicare and the 0.07% PA SUI ($70 + $31.50) and confirmed a total of $11,194. It then submitted $11,097.50, a value that contradicts its own verified sum."
+us,scenario_123,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,It applied only the 7.65% federal FICA to the head's and child's wages. It left out Pennsylvania's mandatory 0.07% employee unemployment compensation withholding ($101.50) on all wages.
+us,scenario_123,payroll_tax,claude-opus-5.5,llm_error,state_local_rule,False,"It treated 7.65% of $145,000 as the whole employee payroll tax. It missed Pennsylvania's 0.07% employee UC contribution on uncapped gross wages, which adds $101.50."
+us,scenario_123,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"It recognized the 0.07% Pennsylvania employee UC contribution but wrongly limited it to the first $10,000 of each worker's wages ($14 total). Pennsylvania's employee contribution has no cap and applies to all $145,000, giving $101.50."
+us,scenario_123,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no employee UI withholding. Pennsylvania requires a 0.07% employee unemployment compensation contribution on all wages, which adds $101.50 to the $11,092.50 of FICA."
+us,scenario_123,payroll_tax,claude-sonnet-5.5,llm_error,state_local_rule,False,"It said no employee state payroll tax applies and so dropped Pennsylvania's mandatory 0.07% employee UC withholding on the $145,000 of wages ($101.50)."
+us,scenario_123,payroll_tax,deepseek-v4-flash-0731,llm_error,state_local_rule,False,"It computed only federal Social Security and Medicare ($11,092.50). It left out Pennsylvania's mandatory 0.07% employee unemployment compensation contribution on all wages ($101.50)."
+us,scenario_123,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"It added up only FICA for the head and child. It did not apply Pennsylvania's 0.07% employee UC withholding to the $145,000 of wages, missing $101.50."
+us,scenario_123,payroll_tax,deepseek-v4-pro-0813,llm_error,state_local_rule,False,"It wrongly concluded that no mandatory state payroll tax applies. Pennsylvania withholds 0.07% of all employee wages for unemployment compensation, which adds $101.50."
+us,scenario_123,payroll_tax,deepseek-v4.1-flash,llm_error,state_local_rule,False,"It computed only 6.2% + 1.45% on $145,000. It omitted Pennsylvania's 0.07% employee unemployment compensation contribution ($101.50)."
+us,scenario_123,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"It counted only the 7.65% federal FICA and then mis-multiplied it: 7.65% × $145,000 is $11,092.50, not $11,108. It also left out Pennsylvania's 0.07% employee UC contribution ($101.50)."
+us,scenario_123,payroll_tax,gemini-3.5-flash-lite,llm_error,state_local_rule,False,"It included only Social Security and Medicare, which come to $11,092.50, not the $11,138 it submitted. It also left out Pennsylvania's mandatory 0.07% employee UC withholding on all wages ($101.50)."
+us,scenario_123,payroll_tax,gemini-3.8-flash,llm_error,state_local_rule,False,It counted only federal FICA on the head's and child's wages. It left out Pennsylvania's 0.07% employee unemployment compensation contribution ($101.50).
+us,scenario_123,payroll_tax,gemini-4-argon,llm_error,state_local_rule,False,It treated 7.65% of household wages as the full payroll tax and left out Pennsylvania's mandatory 0.07% employee UC withholding ($101.50).
+us,scenario_123,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no mandatory employee-side state payroll tax. Pennsylvania withholds 0.07% of gross wages for unemployment compensation, which adds $101.50."
+us,scenario_123,payroll_tax,glm-5.3,llm_error,state_local_rule,False,"It wrongly asserted Pennsylvania has no mandatory employee payroll tax. That drops the 0.07% employee UC contribution on the $145,000 of wages ($101.50)."
+us,scenario_123,payroll_tax,gpt-5.4-mini,llm_error,other,False,"Its stated method (6.2% + 1.45% on $145,000 with no Additional Medicare Tax) gives $11,092.50, but it submitted $12,366, an arithmetic overstatement. It also assumed Pennsylvania has no employee payroll tax and so omitted the $101.50 UC contribution."
+us,scenario_123,payroll_tax,gpt-5.4-nano,llm_error,other,False,"It described 7.65% FICA on $145,000, which is $11,092.50, but submitted $15,399, which its stated rates do not support. It also omitted Pennsylvania's 0.07% employee UC withholding ($101.50)."
+us,scenario_123,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,It computed only 7.65% FICA on each worker's wages. It left out Pennsylvania's mandatory 0.07% employee unemployment compensation contribution ($101.50).
+us,scenario_123,payroll_tax,gpt-6-astra,llm_error,state_local_rule,False,"It included the Pennsylvania employee UC contribution but used an outdated 0.06% rate, giving $87 on $145,000. The current 0.07% rate gives $101.50."
+us,scenario_123,payroll_tax,gpt-6-luna,llm_error,state_local_rule,False,It wrongly said no Pennsylvania employee payroll tax applies. That omits the 0.07% employee UC withholding on all wages ($101.50).
+us,scenario_123,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"It counted only 7.65% FICA and mis-added it: 7.65% × $145,000 is $11,092.50, not $11,107.50. It also left out Pennsylvania's 0.07% employee UC contribution ($101.50)."
+us,scenario_123,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"It wrongly asserted there is no mandatory Pennsylvania employee payroll tax. That drops the 0.07% UC withholding on the $145,000 of wages ($101.50)."
+us,scenario_123,payroll_tax,grok-4.6,llm_error,state_local_rule,False,It wrongly stated Pennsylvania has no mandatory employee-side payroll tax. That omits the 0.07% employee unemployment compensation contribution ($101.50).
+us,scenario_123,payroll_tax,grok-4.7,llm_error,state_local_rule,False,It treated Pennsylvania as having no mandatory employee payroll tax. That leaves out the 0.07% UC withholding on all wages ($101.50).
+us,scenario_123,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"It wrongly ruled out any mandatory state employee payroll tax. That misses Pennsylvania's 0.07% employee UC contribution on $145,000 ($101.50)."
+us,scenario_123,payroll_tax,inkling,llm_error,state_local_rule,False,It set state mandatory employee payroll taxes to zero. Pennsylvania requires a 0.07% employee unemployment compensation contribution on all gross wages ($101.50).
+us,scenario_123,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"It gave no answer for payroll_tax. The correct value is $8,990 SS + $2,102.50 Medicare + $101.50 PA employee UC = $11,194."
+us,scenario_123,payroll_tax,minimax-m3,llm_error,other,False,"It acknowledged that the head's $100,000 and the child's $45,000 of wages are subject to Social Security and Medicare, then submitted 0 anyway. That ignores $11,092.50 of FICA plus $101.50 of Pennsylvania employee UC tax."
+us,scenario_123,payroll_tax,muse-spark-1.3,llm_error,state_local_rule,False,It counted only 7.65% FICA. It left out Pennsylvania's mandatory 0.07% employee UC withholding on the head's and child's wages ($101.50).
+us,scenario_123,payroll_tax,ox-alpha,llm_error,state_local_rule,False,It wrongly said Pennsylvania levies no mandatory employee payroll tax. That omits the 0.07% employee unemployment compensation contribution ($101.50).
+us,scenario_123,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"It wrongly said Pennsylvania has no state payroll taxes. That leaves out the 0.07% employee UC contribution on the $145,000 of wages ($101.50)."
+us,scenario_123,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"It labeled $7,650 as the head's Social Security tax, which double-counts Medicare. It also charged Additional Medicare Tax on '$30,000 above $250,000' even though the household's wages total only $145,000, and it left out the 0.07% Pennsylvania employee UC contribution ($101.50)."
us,scenario_123,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no spouse_medicaid_eligible output, violating the required submission contract and omitting the engine-derived result of 0."
us,scenario_123,spouse_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly stated that the spouse's MAGI was below Pennsylvania's adult Medicaid threshold. The spouse's MAGI is 5.31 times FPL, far above the applicable income limits, and the spouse qualifies through no other Medicaid category."
us,scenario_123,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly found $3,070.06 on the parents' $100,002, then added a separate $1,381.50 child return, even though the dependent's wages are not part of this tax unit's PA income. It then submitted $4,453.06, which does not match its own sum of $4,451.56."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It put the dependent child's $45,000 of wages into the PA base ($145,002). The tax unit's PA taxable income is only the joint filers' $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It made up PA standard deductions of $15,000 and $8,500, which PA does not have, and it also counted the dependent child's $45,000 of wages. With no deductions, the joint filers' $100,002 gives the engine's frozen $3,070.06, and adding the dependent's own return gives the recorded alternative of $4,451.56. This output is excluded from scoring because the definitions do not say which return it covers."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It counted the child's $45,000 of wages in a roughly $145,000 base, then gave $5,900, which is more than 3.07% of that base. Its figure matches neither the engine's frozen $3,070.06, which is 3.07% of the joint filers' $100,002, nor the recorded alternative of $4,451.56. This output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent child's $45,000 of wages to the parents' PA income to get $145,002. The dependent's earnings are not part of the tax unit's $100,002 base. It also submitted $4,452.08, which does not match its own figure of $4,451.56."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It counted Child 1's $45,000 of wages in household PA taxable income ($145,002). The engine's frozen value taxes only the joint filers' $100,002, and this output is excluded from scoring because the definitions do not say whether the dependent's own return counts. It also submitted $4,452.61 instead of its own computed $4,451.56."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly taxed the parents' $100,002 at 3.07%, then added $1,381.50 for the child's wages. Its $4,451.56 is the recorded alternative that counts the dependent's own PA return. The engine's frozen value of $3,070.06 covers only the head and spouse's return, and this output is excluded from scoring because the definitions do not say which return it covers."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, which includes the dependent child's $45,000 of wages and gives the recorded alternative of $4,451.56. The engine's frozen value taxes only the tax unit's $100,002 for $3,070.06, and this output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed the joint return as $100,002 × 0.0307 = $3,070.06. It then switched to including the child's $45,000 on the reasoning that everyone is in one tax unit, but a dependent's wages are not part of that unit's PA taxable income."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly excluded the child's wages and computed $100,002 × 0.0307 = $3,070.06. It then changed the answer to $3,060 citing interest-treatment 'nuances' that do not exist. The $2 of interest is fully taxable and no adjustment applies."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It computed the parents' $3,070.06 correctly but added a separate $1,381.50 child return. Its $4,451.56 is the recorded alternative for the reading that counts the child's own return. The engine's frozen value covers only the head and spouse's return, and this output is excluded from scoring because the definitions do not say which return it covers."
-us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It used $145,002 as the PA base, which includes the dependent child's $45,000 of wages. The tax unit's PA taxable income is $100,002, so the tax is $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the child's $45,000 of wages ($1,381.50) as part of the household liability and left out the $2 of taxable interest. With the interest, the joint filers' $100,002 gives the engine's frozen $3,070.06, and adding the child's return gives the recorded alternative of $4,451.56. This output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, which counts the dependent child's wages. The tax unit's PA income is only the joint filers' $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the $16,100 charitable donation, which PA does not allow as a personal income tax deduction, and it added the child's $45,000 of wages. Without the deduction, its base gives the recorded alternative of $4,451.56, and the engine's frozen value for the joint filers' $100,002 is $3,070.06. This output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It included Child 1's $45,000 in household PA taxable income ($145,002). The dependent's wages are outside the tax unit's base of $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It used a 3% rate instead of PA's 3.07% and applied it to $145,000, which includes the dependent child's wages. At 3.07%, the joint filers' $100,002 gives the engine's frozen $3,070.06, and the household's $145,002 gives the recorded alternative of $4,451.56. This output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, counting the dependent child's $45,000 of wages. The tax unit's PA taxable income is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly taxed the joint filers' $100,002 but added about $1,382 for a separate child return. Its $4,452 is the recorded alternative of $4,451.56 rounded to the dollar. The engine's frozen value of $3,070.06 covers only the head and spouse's return, and this output is excluded from scoring because the definitions do not say which return it covers."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation, and $3,385 equals 3.07% of about $110,260, a base that matches no income figure in the household. It matches neither the engine's frozen $3,070.06, which is 3.07% of the joint filers' $100,002, nor the recorded alternative of $4,451.56. This output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to total household income of $145,002, which includes the dependent child's $45,000 of wages and gives the recorded alternative of $4,451.56. The engine's frozen value taxes only the tax unit's $100,002 for $3,070.06, and this output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It used $145,002 as PA taxable income by adding the dependent child's wages to the parents' income. The tax unit's base is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It counted $145,000 of PA taxable compensation, including Child 1's $45,000. The dependent's wages are not part of the tax unit's income, so compensation is $100,000 and the tax is $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,household_unit_or_filing_status,False,"It explicitly added Child 1's $45,000 of wages to the PA base ($145,002). The tax unit's PA taxable income excludes the dependent's earnings and is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed the joint return as $3,070.06, then added the child's separate $1,381.50. Its $4,451.56 is the recorded alternative that counts the child's own return. The engine's frozen value covers only the joint return, and this output is excluded from scoring because the definitions do not say which return it covers."
-us,scenario_123,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It pooled the dependent child's $45,000 of wages into a $145,002 base. The tax unit's PA taxable income is the joint filers' $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed all household wages including the child's $45,000, then gave $4,807, which is more than 3.07% of $145,002. It matches neither the engine's frozen $3,070.06, which is 3.07% of the joint filers' $100,002, nor the recorded alternative of $4,451.56. This output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It applied federal-style itemized deductions, which PA does not allow, and produced $9,055, about triple the flat-rate result. PA taxes the joint filers' $100,002 at 3.07% for $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's $45,000 of wages in PA taxable income ($145,002). The dependent's earnings are outside the tax unit's $100,002 base, so the tax is $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002, which includes the dependent child's wages, as in the recorded alternative reading. The engine's frozen value taxes only the tax unit's $100,002 for $3,070.06, and this output is excluded from scoring because the definitions do not say whether the dependent's own return counts. It also miscalculated, giving $4,455.06 instead of $4,451.56 for its own base."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002 of wages and interest, counting the dependent child's $45,000. The tax unit's PA base is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the household's full $145,002, including the dependent's $45,000 of wages, which gives the recorded alternative of $4,451.56. The engine's frozen value taxes only the joint filers' $100,002 for $3,070.06, and this output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"It treated all $145,002 of household wages and interest as taxable to the unit, including the dependent child's earnings. The tax unit's PA taxable income is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly taxed the parents' $100,002 but added the child's separate $45,000 of wages. Its $4,451.56 is the recorded alternative that counts the child's own return. The engine's frozen value of $3,070.06 covers only the parents' return, and this output is excluded from scoring because the definitions do not say which return it covers."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,000 of wages, including Child 1's $45,000. The tax unit's taxable wages are only Head's $100,000, so with $2 of interest the tax is $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It counted $145,000 of wages, including the dependent child's $45,000, in the PA base. Only Head's $100,000 of wages plus $2 of interest belongs to the tax unit, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to income that included the dependent child's $45,000 of wages, and its $4,451.06 does not even match 3.07% × $145,002. The engine's frozen value is 3.07% × $100,002 = $3,070.06, and the recorded alternative is $4,451.56. This output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed combined household income of about $145,000, including the dependent child's wages. The tax unit's PA base is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002 of combined compensation and interest, counting Child 1's $45,000. The tax unit's PA taxable income is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly found that PA allows no standard deduction, personal exemptions, or itemized deductions for charity or property tax but used a $145,002 base that includes the dependent child's wages. The tax unit's base is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It treated $145,002 as the unit's PA taxable income by including the dependent child's $45,000 of wages. That base gives the recorded alternative of $4,451.56. The engine's frozen value taxes only the tax unit's $100,002 for $3,070.06, and this output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It explicitly added the child's $45,000 to Head's $100,001 and Spouse's $1. The dependent's wages are not part of the tax unit's PA income, so the tax is 3.07% × $100,002 = $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It added liabilities 'across both filing units' ($100,002 parents + $45,000 child). Its $4,451.56 is the recorded alternative that counts the child's own return. The engine's frozen value of $3,070.06 covers only the parents' return, and this output is excluded from scoring because the definitions do not say which return it covers."
-us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no reasoning, and $2,939.86 equals 3.07% of about $95,760. That base is roughly $4,242 below the tax unit's $100,002, which fits subtracting a deduction PA does not allow. It matches neither the engine's frozen value of 3.07% × $100,002 = $3,070.06 nor the recorded alternative of $4,451.56. This output is excluded from scoring because the definitions do not say whether the dependent's own return counts."
-us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It used $145,002 as PA taxable income, including the dependent child's $45,000 of wages. The tax unit's PA base is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It acknowledged that PA liability is positive but submitted 0 anyway, and it wrongly assumed PA has a standard deduction. The flat 3.07% on the joint filers' $100,002 gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, counting the dependent child's wages. The tax unit's PA taxable income is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly disallowed the charitable, medical and property-tax deductions but included the dependent child's $45,000 of wages in a $145,002 base. The tax unit's base is $100,002, which gives $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It summed Head's and Child 1's wages into $145,000 of taxable compensation. The dependent's $45,000 is not part of the tax unit's PA income, so the tax is 3.07% × $100,002 = $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It used federal taxable income ($67,525) as the PA base, but PA does not start from federal taxable income or allow federal deductions. It then submitted $2,565.30, which does not match its own $2,073.02. PA taxes the joint filers' $100,002 at 3.07% for $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly found the parents' joint-return tax of $3,070.06, then added $1,381.50 for the dependent child's separate return, even though the child's wages are not part of this tax unit's PA taxable income. It also submitted 4,453.06 instead of its own computed 4,451.56."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5.1,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002 by adding the dependent child's $45,000 wages to the parents' $100,002. The tax unit's PA base is only the joint return's $100,002 of compensation and interest, so the tax is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, which includes the dependent child's $45,000 wages. Only the head/spouse joint return's $100,002 counts, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It invented a $15,000 Pennsylvania standard deduction and an $8,500 dependent deduction. Pennsylvania has neither: compensation and interest are taxed with no deductions. It also included the child's $45,000 wages instead of taxing only the joint return's $100,002 at 3.07% ($3,070.06)."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It used a base of about $145,000 that includes the dependent child's wages. Its $5,900 is also not 3.07% of any base it stated. The correct calculation is 3.07% × $100,002 of joint-return compensation and interest = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It added the child's $45,000 wages to the tax unit's PA income, getting $145,002. It then submitted 4,452.08, which does not match its own product of 4,451.56. The joint return's base is $100,002, so the tax is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,prompt_ambiguity,household_unit_or_filing_status,False,"It included Child 1's $45,000 wages in household PA taxable income ($145,002), and its submitted 4,452.61 does not match its own 4,451.56. Only the parents' $100,002 is taxed for this unit: $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed about $3,070 for the head/spouse return but added $1,381.50 for the dependent child's own return. The child's wages are outside this tax unit's PA liability."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, which includes the dependent's $45,000 wages. The tax unit's PA base is the joint return's $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It explicitly found that the joint return's tax is $100,002 × 0.0307 = $3,070.06 and that the child files a separate return. It then wrongly reasoned that a single tax unit means pooling the child's $45,000 into the base."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly left out the child's wages and computed $100,002 × 0.0307 = $3,070.06. It then cut this to $3,060, citing 'interest treatment nuances' that do not exist; PA taxes interest at the same 3.07%."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It computed the parents' $3,070.06 correctly but added $1,381.50 from the dependent child's separate return. The child's wages are not part of this tax unit's PA tax."
+us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-flash-0731,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002 by including the dependent child's $45,000 wages. Only the head/spouse $100,002 is in the tax unit's PA base, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,prompt_ambiguity,household_unit_or_filing_status,False,"It added the dependent child's $1,381.50 to the head's tax and dropped the $2 of interest. The correct figure is 3.07% × $100,002 = $3,070.06 on the joint return only."
+us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro-0813,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, wrongly including the dependent child's wages in the tax unit's PA income. The joint-return base of $100,002 gives $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4.1-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It subtracted the $16,100 charitable donation, but PA allows no charitable or itemized deductions. It also added the child's $45,000 wages. The correct base is the joint return's undeducted $100,002 × 3.07% = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It added Child 1's $45,000 wages to the head's and spouse's income, getting $145,002. The dependent's wages are not part of this tax unit's PA taxable income, so the tax is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It used a 3% rate instead of PA's 3.07%, applied to $145,000 that includes the dependent child's wages. The correct calculation is 3.07% × $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,prompt_ambiguity,household_unit_or_filing_status,False,"It treated $145,002 of total household wages and interest as the tax unit's PA base, including the dependent's $45,000. Only the joint return's $100,002 counts: $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed the joint return's $3,070 but added $1,382 for the child's separate return. The dependent's wages fall outside this tax unit's PA liability."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash-lite,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no derivation. At 3.07%, its $3,385 implies a base of about $110,260, which overstates the joint return's $100,002 of compensation and interest; PA allows no deductions, so the base is exactly $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002 of total household income, which includes the dependent child's wages. The tax unit's PA base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.7-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002 by counting Child 1's $45,000 wages. Only the head/spouse joint return's $100,002 is in the PA base, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.8-flash,prompt_ambiguity,household_unit_or_filing_status,False,"It counted $145,000 of compensation, including the dependent child's $45,000. The tax unit's PA compensation is only the head's $100,000 plus $2 of interest, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-4-argon,prompt_ambiguity,household_unit_or_filing_status,False,"It explicitly added Child 1's $45,000 wages into the PA base of $145,002. The dependent's earnings do not belong to the parents' joint return, so the tax is $100,002 × 3.07% = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly computed the joint return's $3,070.06 and noted that the child files separately. It then added the child's $1,381.50, which is not part of this tax unit's PA liability."
+us,scenario_123,state_income_tax_before_refundable_credits,glm-5.3,prompt_ambiguity,household_unit_or_filing_status,False,"It summed $100,000 + $45,000 + $2 as the tax unit's PA income, including the dependent's wages. The correct base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,prompt_ambiguity,household_unit_or_filing_status,False,"It included the child's wages, and its $4,807 implies a base of about $156,580, more than even the pooled $145,002. The correct calculation is 3.07% × the joint return's $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,prompt_ambiguity,household_unit_or_filing_status,False,"It applied a federal-style itemized-deduction framework that PA does not have. Its $9,055 is roughly triple what PA's flat 3.07% produces. Applied to the joint return's $100,002, the correct tax is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent child's $45,000 wages in the PA base of $145,002. Only the head/spouse $100,002 is taxed for this unit, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed the household's $145,002, including the child's wages, and its 4,455.06 also does not equal 3.07% × $145,002. The correct figure is 3.07% × $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002 of wages and interest, which includes the dependent's $45,000. The tax unit's base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It used household-wide wages of $145,002, including Child 1's earnings. The dependent's wages are not part of the parents' PA taxable income, so the tax is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-astra,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002 of household wages and interest instead of the joint return's $100,002. Adding the dependent child's $45,000 raised the tax above the correct $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-luna,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly separated the parents' $100,002 but added the tax on the child's separate $45,000 return. This tax unit's PA liability is only $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-6-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,000 of wages that includes the dependent child's $45,000. Only the head's $100,000 wages plus $2 interest belong to the tax unit's base: $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-6.1-sol,prompt_ambiguity,household_unit_or_filing_status,False,"It counted $145,000 of wages by including Child 1's earnings. The joint return's PA base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-6.1-terra,prompt_ambiguity,household_unit_or_filing_status,False,"It pooled the dependent child's $45,000 wages into the PA base, and its 4,451.06 is off by $0.50 from its own pooled product. The correct tax is 3.07% × $100,002 = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to about $145k of combined household income, including the dependent's wages. The tax unit's PA base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,prompt_ambiguity,household_unit_or_filing_status,False,"It taxed $145,002, pooling the dependent child's $45,000 wages with the parents' income. The joint return's $100,002 yields $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-4.6,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly allowed no deductions but used $145,002, which includes Child 1's wages. The tax unit's base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-4.7,prompt_ambiguity,household_unit_or_filing_status,False,"It included the dependent's $45,000 wages in taxable income of $145,002. Only the head/spouse joint return's $100,002 counts for this tax unit, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,prompt_ambiguity,household_unit_or_filing_status,False,"It explicitly added the child's $45,000 to the head's and spouse's income, getting $145,002. The dependent's wages are not part of this tax unit's PA taxable income, so the tax is $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,inkling,prompt_ambiguity,household_unit_or_filing_status,False,"It summed the tax across both filing units: the parents' $100,002 and the child's separate $45,000. The requested liability is the tax unit's joint return only: $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,prompt_ambiguity,household_unit_or_filing_status,False,"It gave no reasoning. At 3.07%, its $2,939.86 implies a base of about $95,760, meaning it subtracted some deduction from the $100,002; PA allows none against compensation and interest, so the full $100,002 is taxed for $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly applied no deductions but used $145,002, which includes the dependent child's wages. The tax unit's base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,prompt_ambiguity,household_unit_or_filing_status,False,"It stated that PA tax liability is positive but submitted 0. It also wrongly assumed a PA standard deduction. The flat 3.07% on the joint return's $100,002 gives $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,muse-wonder,prompt_ambiguity,household_unit_or_filing_status,False,"It applied 3.07% to $145,002, including the dependent's $45,000 wages. The tax unit's PA base is $100,002, giving $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,ox-alpha,prompt_ambiguity,household_unit_or_filing_status,False,"It correctly disallowed charitable, medical and property tax deductions but included the child's $45,000 wages in the $145,002 base. The joint return's $100,002 yields $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,prompt_ambiguity,household_unit_or_filing_status,False,"It added Child 1's $45,000 wages to the head's wages and the couple's interest, getting $145,002. The dependent's earnings are outside this tax unit's PA base, so the tax is $100,002 × 3.07% = $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,qwen3.8-max,prompt_ambiguity,household_unit_or_filing_status,False,"It used federal taxable income ($67,525) as the PA base. PA does not start from federal taxable income and allows no standard or itemized deductions; it taxes the full $100,002 of compensation and interest at 3.07%. It also submitted 2,565.30, which contradicts its own $2,073.02."
us,scenario_123,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
us,scenario_123,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv
index 581fd417..ad0810af 100644
--- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv
+++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv
@@ -1,5 +1,5 @@
country,scenario_id,variable,wrong_model_count,case_failure_sources,case_failure_subtypes,reference_suspect,reference_bug_hypothesis,case_annotation
-us,scenario_000,federal_income_tax_before_refundable_credits,46,llm_error,taxable_income_or_deductions,False,,"With no dependent child, a surviving spouse cannot use qualifying-surviving-spouse status, so the head files single. Taxable Social Security under the $25,000/$34,000 thresholds is $18,542.70, which is below the 85% cap. That makes AGI $50,437.11. Because OBBBA made TCJA permanent, 2026 has no personal exemption, and the single standard deduction is $16,100 plus the $2,050 aged add-on, for $18,150. That beats itemizing $17,587.20, and the $6,000 senior deduction is added on top, so taxable income is $26,287.11. Tax is 10% on the first $12,400 plus 12% on the rest, or $2,906.45. Wrong answers came from four main errors: using joint/QSS deductions or brackets, assuming TCJA expired (personal exemption and a 15% bracket), leaving out the $6,000 senior deduction, or miscomputing taxable Social Security."
+us,scenario_000,federal_income_tax_before_refundable_credits,47,llm_error,household_unit_or_filing_status,False,,"The head is a widowed person with no dependents. Qualifying-surviving-spouse status requires a dependent child, so the head files as Single. Taxable Social Security is $18,542.70 under the $25,000/$34,000 thresholds, which makes AGI $50,437.11. Deductions are the 2026 single standard deduction of $16,100, the $2,050 aged add-on and the OBBBA $6,000 senior deduction. That leaves $26,287.11 of taxable income, taxed at 10% up to $12,400 and 12% above, for $2,906.45. The wrong answers fall into three groups. Some took the joint-sized QSS standard deduction (and sometimes the joint Social Security thresholds). Some assumed the TCJA had expired, bringing back personal exemptions and a 15% bracket. Others left out the $6,000 senior deduction or misapplied the second tier of the Social Security taxation formula."
us,scenario_000,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_000,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Texas Medicaid eligibility at age 77 requires qualification through an aged Medicaid pathway; age alone does not establish a qualifying category. This person has no Medicaid eligibility category, receives no SSI, and has MAGI income of 3.98 times FPL, so the immigration-status requirement does not create eligibility by itself."
us,scenario_000,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wage and salary compensation, not to Social Security retirement benefits, pensions, IRA distributions, interest, or veterans benefits. Because the household reports no wages or other employee compensation, the employee Social Security and Medicare tax base is zero and payroll tax is $0."
@@ -13,7 +13,7 @@ us,scenario_001,state_income_tax_before_refundable_credits,7,llm_error,state_loc
us,scenario_002,federal_income_tax_before_refundable_credits,11,prompt_ambiguity,taxable_income_or_deductions,False,,"PolicyEngine does not count the $24,000 disability benefits or the $48,000 veterans benefits in gross income. That leaves $18,740 of other income: $18,340 of pension plus $400 of interest. Provisional income is $18,740 plus half of Social Security ($22,040), or $40,780, which is below the $44,000 MFJ second-tier threshold. Only 50% of the excess over $32,000 is taxable, so $4,390 of Social Security is taxable and AGI is $23,130. The 2026 MFJ standard deduction with both spouses 65+ is $35,500, before the $6,000-per-senior deduction, so it wipes out all of that income and tax is $0. Most models counted the disability benefits as taxable, made all of Social Security taxable, or both, and then also understated the deductions. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 1,890.30. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)."
us,scenario_002,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to covered wages and other employment compensation, not Social Security retirement benefits, pensions, disability benefits, veterans benefits, or taxable interest. Because the household has no wages or other payroll-taxable earnings, every employee-side payroll-tax component is zero."
us,scenario_002,spouse_medicaid_eligible,1,llm_error,household_unit_or_filing_status,False,,"Medicaid eligibility for the spouse cannot be established by treating the spouse's lack of individually reported income as zero countable income. The spouse qualifies through no Washington Medicaid pathway: the engine assigns category NONE, records no SSI, and calculates household MAGI at 2.90 times FPL, above the applicable MAGI limits."
-us,scenario_003,federal_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"AGI is $183,290.25: 85% of Social Security is taxable, the net capital loss is $1,582, and there are no wages, so the 401(k)/IRA contributions cannot be deducted. Under OBBBA-permanent TCJA law, the 2026 MFJ standard deduction is $32,200 and neither spouse is 65. That leaves $147,330.25 of ordinary income, taxed through the 2026 brackets (10% to $24,800, 12% to $100,800, 22% above), for $21,836.65, plus 15% on $3,760 of qualified dividends ($564), totaling $22,400.65. Most models went wrong in one of four ways: they assumed the TCJA sunset (personal exemptions and 10/15/25% brackets), used 2025 or estimated bracket and standard-deduction parameters, took deductions that don't apply (the $500 auto-loan interest, or retirement contributions without compensation), or made arithmetic slips in AGI or taxable income."
+us,scenario_003,federal_income_tax_before_refundable_credits,40,llm_error,thresholds_rates,False,,"Because there are no wages, the 401(k) and IRA contributions are not deductible. AGI is $183,290.25: $132,707 of non-Social Security income after the net $1,582 capital loss, plus 85% of Social Security. Subtracting OBBBA's 2026 joint standard deduction of $32,200 leaves $151,090.25 of taxable income, with no age-65 add-on, no senior deduction and no auto-loan-interest deduction. Ordinary income of $147,330.25 is taxed at the 2026 OBBBA brackets (10% to $24,800, 12% to $100,800, then 22%), and $3,760 of qualified dividends are taxed at 15%. The biggest group of misses assumed the TCJA had sunset, so they used personal exemptions and 10/15/25% brackets. Most other misses used stale 2025 standard-deduction or bracket figures, took deductions the household does not qualify for, or made arithmetic slips with the capital loss."
us,scenario_003,head_medicare_eligible,4,llm_error,age_disability,False,,"Medicare eligibility without disability or ESRD requires age 65. The head is 64 for the whole 2026 tax year, and the prompt says ages and statuses stay constant all year. The head is listed with no disability, no SSDI, and no ESRD, so no non-age pathway applies. Two models correctly reasoned ""64 < 65, not eligible"" but still submitted 1. The other two invented a ""near-term"" or ""projected"" eligibility rule that doesn't exist."
us,scenario_003,payroll_tax,2,llm_error,payroll_tax_base,False,,"Usual weekly hours and employer-sponsored insurance establish work and coverage facts but do not supply wage income or an hourly rate. Because unlisted numeric inputs are zero, both people have zero wages subject to employee Social Security and Medicare taxes, so employee payroll tax is zero."
us,scenario_003,state_income_tax_before_refundable_credits,1,llm_error,state_local_rule,False,,"Texas imposes no state individual income tax, and exclusive Texas residency makes every other state's income-tax component zero. The aggregate state income tax before refundable credits is therefore the sum of 51 zero components, yielding $0."
@@ -24,18 +24,18 @@ us,scenario_004,payroll_tax,1,llm_error,payroll_tax_base,False,,"Payroll tax app
us,scenario_004,spouse_medicaid_eligible,1,llm_error,taxable_income_or_deductions,False,,"New York's ACA expansion pathway requires the spouse's MAGI income level to be at or below 138% of FPL. The household's applicable MAGI is 2.39 times FPL, and the 51-year-old spouse has no disability, blindness, SSI, age-based, dependent, or other categorical pathway, so the Medicaid category is NONE."
us,scenario_004,state_income_tax_before_refundable_credits,5,llm_error,taxable_income_or_deductions,False,,"Both spouses show 40-hour work weeks but no wage amount, and the prompt directs that unlisted numeric inputs be treated as 0, so the household's only non-Social-Security income is $1,200 of taxable interest less a $135 partnership loss = $1,065. Provisional income is $1,065 + half of $51,475 = $26,803, below the $32,000 married-filing-jointly base, so none of the Social Security benefit is federally taxable, and New York subtracts any taxable Social Security from federal AGI regardless. New York AGI of $1,065 is entirely absorbed by the ~$16,050 MFJ standard deduction, leaving $0 of NY taxable income and $0 of tax before refundable credits. Every wrong answer either taxed the $51,475 Social Security benefit, imputed phantom wages from the hours-worked fact, or both."
us,scenario_004,state_refundable_credits,1,llm_error,state_local_rule,False,,"This couple has no wages or rent, so its federal AGI is only about $1,065: $1,200 of interest minus a $135 partnership loss. None of the $51,475 in Social Security is taxable, because provisional income is below the $32,000 joint base. The only NY credit this low income could trigger is the household credit under Tax Law §606(b), and that credit is nonrefundable. It can only reduce NY tax, which is already zero, and it never counts toward state_refundable_credits. None of New York's refundable credits apply either: there is no federal EITC, no qualifying child, and no property tax or rent for the real property tax credit, so the refundable total is $0."
-us,scenario_005,federal_income_tax_before_refundable_credits,51,reference_engine_defect,taxable_income_or_deductions,False,"The reference AGI of $536,764.52 (taxable income $503,494.75 plus $33,269.77 of deductions) equals pre-IRA income of $538,928 ($388,350 + $65,936 + $87,642 − $3,000) minus the full $2,164 traditional IRA deduction. That contradicts IRC §219(g): both spouses make 401(k) elective deferrals, so they are active participants, and the 2026 MFJ phase-out of $129,000–$149,000 reduces the deduction to $0 at this MAGI. PolicyEngine does not apply the active-participant phase-out, which understates tax by about $692.","PolicyEngine taxes 2026 under OBBBA's permanent TCJA structure. The frozen reference uses $536,764 of AGI, with $41,650 of pro rata traditional 401(k) deferrals, no state-refund income and a defective $2,164 IRA deduction. Its $32,200 standard deduction plus $1,070 non-itemizer cash charitable deduction leave $503,494.75 taxable. Ordinary brackets apply to $425,574.75 after removing qualified dividends, producing $89,095.92 of ordinary tax, plus $11,688 on $77,920 of qualified dividends and $5,721.98 NIIT. The policyengine-us 2.60.0 engine applies the IRA phase-out and includes the refund, giving the rechecked $108,525.58. The output remains excluded; the record leaves refund taxation, excess-deferral allocation and NIIT inclusion unresolved. Other model errors include TCJA-sunset assumptions, invented mortgage interest, subtracting employer premiums from wages and omitting the cash charitable deduction. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 106,505.90; the recorded correction is 107,198.34 under the refund-excluded reading. The frozen engine omitted the IRA active-participant phase-out. The policyengine-us 2.60.0 engine applies that phase-out and includes the refund, giving 108,525.58. The output remains excluded because refund taxation, excess-deferral allocation and NIIT inclusion remain unresolved. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
+us,scenario_005,federal_income_tax_before_refundable_credits,52,reference_engine_defect,taxable_income_or_deductions,False,"The $5,163.84 above-the-line total is the $3,000 capital-loss deduction plus the full $2,164 of traditional IRA contributions. IRC §219(g) phases that deduction out for active participants in an employer plan, and for 2026 joint filers the phase-out runs from $129,000 to $149,000 MAGI. Both spouses make 401(k) elective deferrals and MAGI is about $537k, so PolicyEngine appears to skip the active-participant phase-out, overstating deductions by $2,164 (about $692 of tax).","The reference has three parts: $89,095.92 of ordinary tax on $425,574.75 at the 2026 OBBBA joint-filer brackets, $11,688 from taxing $77,920 of qualified dividends at 15%, and $5,721.98 of NIIT (3.8% of $150,578, which is net of the $3,000 capital loss). Taxable income of $503,494.75 depends on four steps. Traditional 401(k) deferrals are prorated to the $24,500 combined traditional-plus-Roth limit ($20,825 each, so taxable wages are $388,350). The $4,148 state refund is left out of gross income. The deduction is the $32,200 standard deduction plus the $1,070 OBBBA deduction for non-itemizers' cash gifts. Models went wrong mainly by assuming the TCJA expired in 2026 (Pease, personal exemptions, pre-2018 brackets, AMT), inventing mortgage interest from the loan balance, subtracting employer health premiums from wages, leaving out NIIT, or adding back the state refund and the full 401(k) deferrals. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 106,505.90; the recorded correction is 107,198.34 under the refund-excluded reading. The frozen engine omitted the IRA active-participant phase-out. The policyengine-us 2.60.0 engine applies that phase-out and includes the refund, giving 108,525.58. The output remains excluded because refund taxation, excess-deferral allocation and NIIT inclusion remain unresolved. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
us,scenario_005,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_005,payroll_tax,46,llm_error,state_local_rule,False,,"The federal employee FICA components are simple: Social Security is 6.2% × ($180,000 + $184,500 2026 cap) = $22,599, Medicare is 1.45% × $430,000 = $6,235, and the Additional Medicare Tax is 0.9% × ($430,000 − $250,000 MFJ) = $1,620. Together they come to $30,454. The step that separated the answers was California SDI. For 2026 the rate is 1.3% with no wage ceiling, which gives $5,590 and a total of $36,044. Most models dropped SDI, used the old 1.1% or 1.2% rate, or applied the repealed SDI wage cap. Some also used an outdated Social Security wage base, or subtracted employer-sponsored insurance premiums from the gross wages used for FICA and SDI."
+us,scenario_005,payroll_tax,47,llm_error,state_local_rule,False,,"The reference applies payroll taxes to gross wages of $430,000, with nothing subtracted for 401(k) deferrals or ESI premiums. Social Security is 6.2% of $180,000 plus the spouse's wages up to the 2026 wage base of $184,500, which comes to $22,599. Medicare is 1.45% of $430,000, or $6,235, and the Additional Medicare Tax is 0.9% of the $180,000 above the $250,000 joint threshold, or $1,620. California SDI is 1.3% of all $430,000 with no wage cap, or $5,590. Most models got FICA right but missed the SDI piece: they left it out, used the older 1.1% or 1.2% rate, or applied the per-worker wage cap that ended in 2024. A second group shrank the wage base by subtracting premiums or deferrals, and others used an outdated Social Security wage base."
us,scenario_005,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_005,state_income_tax_before_refundable_credits,51,llm_error,taxable_income_or_deductions,False,,"The engine starts from federal AGI of $543,076 and subtracts only the $4,147.68 SALT refund, giving CA AGI of $538,928.31. It then itemizes property tax ($17,280), charity ($2,573.53) and employee business expenses above the 2% floor ($2,986.30), less a $2,319.90 high-income cutback. That leaves $20,519.93: there is no mortgage interest because none is listed, no SALT cap, and no exclusion of employer premiums or IRA contributions. Taxable income of $518,408.38 has a top marginal rate of 9.3% and gives $41,089.25 of tax, minus a phased-down $114 exemption credit. Near misses mostly used CA AGI of $534,270 or lower, skipped the misc deduction or the cutback, or used larger exemption credits. Far misses deducted unlisted mortgage interest, taxed the exempt pension, used the standard deduction, or got the bracket arithmetic wrong."
+us,scenario_005,state_income_tax_before_refundable_credits,52,llm_error,taxable_income_or_deductions,False,,"California tax starts from the engine's federal AGI of $543,076. Subtracting the $4,148 SALT refund gives a CA AGI of $538,928, which is higher than the $534,270 most models built. California itemized deductions are $22,840: $17,280 real estate tax, $2,574 charity and $2,986 of unreimbursed employee expenses above the 2% floor, which CA still allows. No mortgage interest is deductible because only a loan balance was listed. The 6% high-income limitation cuts $2,320, leaving $20,520 of deductions and $518,408 of taxable income. That produces $41,089 of tax, and the exemption credit is phased down to just $114. Most models landed within about $1–2k. They went wrong by understating AGI, omitting the 2%-floor employee expenses or the limitation, imputing mortgage interest, or misjudging the exemption-credit phaseout and the 2026 brackets."
us,scenario_005,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_007,federal_income_tax_before_refundable_credits,42,llm_error,taxable_income_or_deductions,False,,"The correct path has four steps. First, provisional income is $35,280 + $9,642.50 = $44,922.50; the $284 tax-exempt pension is not added. Second, taxable Social Security is the lesser of 85% of benefits ($16,392.25) and 0.85×($44,922.50−$34,000) + $4,500 = $13,784.12, so AGI is $49,064.12. Third, the 2026 single standard deduction under the OBBBA (which kept the TCJA structure permanently) is $16,100, giving taxable income of $32,964.12. Fourth, the 10% bracket ends at $12,400, so tax is $1,240 + 12%×$20,564.12 = $3,707.70, with no nonrefundable credits. The models missed in five ways: they used older or guessed 2026 deductions and brackets, applied TCJA-sunset rules (a personal exemption and a 15% bracket), took the 85% cap instead of the lesser-of formula, added the tax-exempt pension to provisional income, or left Social Security out of income altogether. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The judge labeled seven models' rows reference_later_law because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, a personal exemption and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors."
us,scenario_007,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_007,head_medicare_eligible,23,prompt_ambiguity,age_disability,False,,"The head is 56, so the age-65 pathway is closed. Disability-based Medicare requires 24 months of SSDI entitlement (or ESRD/ALS). PolicyEngine tracks this with a months-receiving-SSDI input that is unlisted here, so it defaults to 0, and no disability or ESRD flag is set. Every model treated a positive Social Security disability income amount as proof of Medicare entitlement, and several assumed the 24-month waiting period was met. The prompt only says facts are constant within the tax year, which does not establish 24 prior months of receipt, so the head is not Medicare eligible. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it."
us,scenario_007,payroll_tax,2,llm_error,payroll_tax_base,False,,"Payroll tax applies to wages and other FICA-covered earned compensation, not to Social Security disability benefits or private pension income. The prompt specifies no wage or self-employment income and requires unlisted numeric inputs to be zero, so the employee payroll-tax base is zero and payroll tax is $0."
us,scenario_007,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_007,state_income_tax_before_refundable_credits,51,llm_error,taxable_income_or_deductions,False,,"The Idaho calculation has five steps: start from federal AGI ($49,064.12), subtract the $13,784.12 of taxable Social Security and the $2,080 Idaho health insurance premium subtraction to reach Idaho AGI of $33,200, take the $16,100 conforming standard deduction to reach $17,100 of taxable income, apply 5.3% only above the $4,811 zero-rate amount ($651.32), then add the $10 permanent building fund tax. Many models used $19,180 because they skipped the $2,080 health premium subtraction. Many also taxed every dollar at a flat rate with no zero bracket, used the outdated 5.695% or 5.8% rates, or left out the $10 PBF. A smaller group never subtracted Social Security, or wrongly zeroed out the tax by applying a retirement exclusion this 56-year-old with a private pension does not qualify for."
+us,scenario_007,state_income_tax_before_refundable_credits,52,llm_error,state_local_rule,False,,"The correct 2026 Idaho calculation has five steps. Start from federal AGI ($49,064.12), subtract all taxable Social Security ($13,784.12), and also subtract the $2,080 of health insurance premiums the head paid, which gives Idaho AGI of $33,200. Take off the conforming $16,100 single standard deduction to get $17,100 of taxable income. Tax that at 5.3% above the roughly $4,811 zero-rate amount ($651.32), then add the $10 permanent building fund tax, for $661.32. Most models dropped the $2,080 premium subtraction, the zero-rate bracket, or the $10 building fund tax. Many also used the old 5.695% or 5.8% rate, a wrong standard deduction, or never subtracted Social Security."
us,scenario_007,state_refundable_credits,40,llm_error,state_local_rule,False,,"The only refundable Idaho credit here is the Idaho Grocery Credit. Under the 2025 Idaho legislation, it is $155 per person for an individual under 65 starting in tax year 2025. The credit has no income phase-out, and the head does not get SNAP, so all 12 months qualify. That gives the full $155. Wrong answers fall into two groups. One group used outdated per-person amounts: $120 (the 2023–2024 amount), $100 (the pre-2023 amount) or $140 (the old senior amount, or a guessed inflation adjustment). The other group set the credit to $0, either by calling it nonrefundable, inventing an income phase-out, or saying Idaho has no such credit."
us,scenario_008,child1_chip_eligible,9,llm_error,health_coverage,False,,"CHIP is a residual program: it covers children who do NOT qualify for Medicaid, so its income band has a floor at the state's children's Medicaid limit, not just a ~350% FPL ceiling. With household income of about $30,915 for a family of eight — roughly 54% of FPL — the 17-year-old clears New Jersey's Medicaid test in the OLDER_CHILD category, and that Medicaid eligibility forecloses CHIP. All nine models tested only the upper CHIP ceiling and read very low income as evidence for CHIP, when low income is exactly what routes the child to Medicaid instead."
us,scenario_008,child1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"A 17-year-old tax-unit dependent falls within New Jersey's OLDER_CHILD Medicaid category. The household's MAGI is 0.55 times FPL, below the category's income threshold, so the child is eligible without any additional eligibility trigger."
@@ -52,7 +52,7 @@ us,scenario_008,child4_medicaid_eligible,1,llm_error,categorical_eligibility,Fal
us,scenario_008,child4_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC child eligibility is limited to children under age five, so a 10-year-old is categorically ineligible. The household’s approximately $30,915 of income also exceeds the applicable WIC income ceiling in PolicyEngine’s computation, providing an independent income-based disqualification."
us,scenario_008,child5_chip_eligible,11,llm_error,health_coverage,False,,"CHIP is residual coverage: a child who qualifies for Medicaid is barred from CHIP, so the binding step is the Medicaid screen, not the CHIP ceiling. Child5 is 5 years old, and household income of about $30,915 for an 8-person unit (~54-59% FPL) sits far below New Jersey's Medicaid threshold for the under-6 (YOUNG_CHILD) age band, so the child is Medicaid-eligible and therefore CHIP-ineligible. Every wrong model ran a single test — is income below NJ FamilyCare's ~355% FPL child limit — and answered yes, inverting the logic: the further below that ceiling the household sits, the more certainly Medicaid captures the child and forecloses CHIP. None applied the Medicaid-first crowd-out or the separate, lower under-6 Medicaid income band that the child's age triggers."
us,scenario_008,child5_early_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Early Head Start serves infants and toddlers under age 3, while Head Start is the distinct program for preschool-age children. Child 5 is age 5 and therefore fails the Early Head Start age requirement regardless of household income or housing-assistance receipt."
-us,scenario_008,child5_head_start_eligible,5,llm_error,age_disability,False,,"Every model rejected Child 5 on age alone, treating preschool Head Start as serving only 3- and 4-year-olds. Under 45 CFR 1302.12(b)(1) the range runs from age 3 up to the age of compulsory school attendance, which is 6 in New Jersey, and PolicyEngine's Head Start age window includes 5-year-olds. The age test is met. The child is also income-eligible, both categorically through the household's SNAP receipt and directly because household income of about $30.6K is well under the poverty-guideline threshold for a family of 8. So the answer is eligible. No model reached the income or categorical test, because each stopped at an age cutoff that was set too low."
+us,scenario_008,child5_head_start_eligible,6,llm_error,age_disability,False,,"Every model capped Head Start preschool eligibility at ages 3–4, so they ruled out the 5-year-old. Under 45 CFR 1302.12(b), a child stays eligible from age 3 until the age required to attend school, and PolicyEngine's Head Start age range includes 5-year-olds. Once age is not a bar, the child qualifies two ways: the household gets SNAP (about $1,260/month), which is a categorical pathway, and household income of about $30.6K is well under the poverty-guideline income test for a family of 8. No model got past the age gate to apply the SNAP categorical pathway or the income test."
us,scenario_008,child5_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"A five-year-old in New Jersey falls within the YOUNG_CHILD Medicaid category. At MAGI equal to 0.55 times FPL, the child satisfies that category's income test and is Medicaid eligible."
us,scenario_008,child5_wic_eligible,13,llm_error,age_disability,False,,"WIC's child category covers children from age 1 up to their 5th birthday, so ages 1 through 4. A child whose age is listed as 5 has already turned 5 and has aged out. Income does not decide this case: the age-1 Child 6 is WIC-eligible in the same household, so the household meets the income test. All 13 models treated age 5 as still inside the WIC age window, reading 'under 5' or 'up to the 5th birthday' as if it included age 5."
us,scenario_008,child6_chip_eligible,12,llm_error,health_coverage,False,,"CHIP is residual coverage: PolicyEngine returns is_chip_eligible False for any child who already qualifies for Medicaid. The 1-year-old in this eight-person New Jersey household, with income near $30,915 (roughly 54% FPL), falls squarely inside the YOUNG_CHILD Medicaid category, so the Medicaid flag is 1 and the CHIP flag is 0. All twelve models tested only the upper NJ FamilyCare CHIP ceiling (300–355% FPL), found the household far beneath it, and treated that as proof of CHIP eligibility — when clearing that ceiling by such a margin is exactly what places the child under the lower young-child Medicaid threshold. Several went further and merged Medicaid and CHIP into a single ""NJ FamilyCare"" eligibility, erasing the mutual exclusivity that decides this output."
@@ -65,16 +65,16 @@ us,scenario_008,free_school_meals_eligible,1,llm_error,categorical_eligibility,F
us,scenario_008,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"New Jersey expanded Medicaid under the ACA. Any non-elderly adult with household MAGI at or below 138% FPL qualifies in the ADULT category, with no disability requirement. This household has 8 members. Its MAGI is about $30.8k: $26,800 in wages, plus $4,000 in self-employment income minus half of SE tax, plus $115 in interest. That works out to 0.55× FPL, well under the 138% limit, so the 42-year-old head is eligible. Every model answered No for one of three reasons: it assumed non-disabled adults are categorically excluded, it misjudged where the income falls against the threshold, or it made up an exclusion rule."
us,scenario_008,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC categorical eligibility is determined person by person, not conferred on every household member by the presence of an eligible infant or young child. The 42-year-old Head is not pregnant, postpartum, or breastfeeding and is not an infant or child under age 5, so satisfying the household income limit does not make the Head eligible."
us,scenario_008,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_008,payroll_tax,49,llm_error,state_local_rule,False,,"Federal employee FICA is 7.65% of the spouse's $26,800 wages, or $2,050.20, and most models got it right. gpt-5.4-nano's total of $1,525 falls below even that amount, gpt-5.4-mini computed FICA correctly but then reported an unsupported $4,097, and glm-5.2 and kimi-k2.6 returned no answer. The answers split on New Jersey's mandatory employee contributions, which total $226.46 at the 2026 worker rates: TDI 0.19% ($50.92), FLI 0.23% ($61.64), UI 0.3825% ($102.51) and Workforce/SWF 0.0425% ($11.39), all applied to $26,800 of taxable wages. Most models left the NJ contributions out entirely. Others applied wrong or outdated NJ rates: a 0% TDI rate, a 0.09% FLI rate, or the 2025 rates of 0.23% TDI and 0.33% FLI. Some also dropped the UI or workforce components, which pushed the total below $2,276.66. Models that used the 2025 TDI and FLI rates or another inflated combined rate landed above it."
+us,scenario_008,payroll_tax,50,llm_error,state_local_rule,False,,"Every model got employee FICA right: 7.65% of the spouse's $26,800 wages is $2,050.20, and no Additional Medicare Tax applies. The answers split on New Jersey's four mandatory employee contributions, which all apply to the full $26,800 for 2026: TDI at 0.19% ($50.92), FLI at 0.23% ($61.64), UI at 0.3825% ($102.51), and the workforce fund at 0.0425% ($11.39). Together these add $226.46. Most models either left out the NJ contributions entirely or used old or guessed TDI/FLI rates, such as TDI at 0%, 0.10% or 0.23%, and FLI at 0.09% or 0.33%. Some also missed the separate 0.0425% workforce-fund contribution."
us,scenario_008,reduced_price_school_meals_eligible,1,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is a mutually exclusive lower tier, not a generic indicator that income is below the 185% FPL ceiling. This household is at 55% of the federal poverty guideline and also meets categorical eligibility, so it receives the superior free-meals tier and reduced_price_school_meals_eligible equals 0."
us,scenario_008,self_employment_tax,4,llm_error,payroll_tax_base,False,,"Self-employment tax is 15.3% of 92.35% of the $4,000 net self-employment income: $4,000 × 0.9235 × 0.153 = $565.18. The deduction for one-half of self-employment tax is an income-tax deduction and does not reduce the self-employment tax liability itself."
-us,scenario_008,snap,50,llm_error,thresholds_rates,False,,"This is a plain SNAP calculation for an 8-person household. Gross income is $2,576.25 a month. Subtract the FY2026 6+ person standard deduction of $299 and the 20% earned income deduction of $513.33 to get $1,763.92, which rounds to $1,764. There is no shelter deduction because the $579 rent is below half of net income, no medical deduction because $25/month of OTC costs is under the $35 threshold, and no asset test because NJ grants categorical eligibility through TANF non-cash benefits. Thirty percent of $1,764 is $529.20, which rounds up to a $530 contribution, so the benefit is $1,789 − $530 = $1,259 a month. Most models went wrong in one of four ways: they used old (FY2025) or invented maximum allotments and standard deductions, made up shelter or medical deductions, applied an asset test that categorical eligibility removes, or skipped SNAP's rounding of net income and the contribution."
+us,scenario_008,snap,51,llm_error,thresholds_rates,False,,"The reference uses FY2026 SNAP parameters: a $1,789 maximum allotment for 8 people and a $299 standard deduction for households of 6 or more. Gross income is $2,576.25. Subtracting the 20% earned income deduction ($513.33) and the standard deduction gives $1,763.92, which rounds to net income of $1,764. The expected contribution is 30% of that ($529.20), rounded up to $530, so the benefit is $1,259 a month. NJ broad-based categorical eligibility removes the asset test, rent is less than half of adjusted income, and the over-the-counter expenses are under the $35 medical threshold, so there are no shelter or medical deductions. Wrong models fall into four groups: some used stale or wrong allotment and deduction figures, some subtracted the standard deduction only once against annual income, some skipped the whole-dollar rounding and the round-up of the contribution, and some zeroed out the benefit through an asset test or income test that does not apply."
us,scenario_008,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"New Jersey adopted the ACA adult expansion, so a non-elderly adult 19-64 who is not a tax-unit dependent qualifies on the MAGI pathway at or below 138% FPL, with no categorical hook, no asset test, and no benefit-receipt disqualifier. The test compares household MAGI to the FPL for the full household size: roughly $30,900 of wages, self-employment income and interest against the eight-person FPL is 0.55 x FPL, less than half the 1.38 limit. Housing assistance is excluded from MAGI and the $69,500 in bank accounts is irrelevant because MAGI categories carry no resource test. Each wrong answer either skipped this income test entirely or overrode it with a disqualifier that does not exist."
us,scenario_008,spouse_medicare_eligible,6,llm_error,age_disability,False,,"Medicare eligibility here turns on a single age test: the person must be 65 or older, and the Spouse is 41. The trap is the `is_disabled` flag: every wrong model converted it into a disability-based Medicare pathway, but real-world under-65 Medicare entitlement runs through 24 months of SSDI cash benefits (or ESRD/ALS), none of which is present — the household reports no Social Security disability income, and the prompt directs that unlisted income and status inputs be treated as 0/false. A disability flag alone establishes neither SSDI entitlement nor the 24-month qualifying period, so it cannot produce Medicare eligibility for a 41-year-old."
us,scenario_008,spouse_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC eligibility is person-specific: an adult must be pregnant, breastfeeding, or within the applicable postpartum period, while infants and children qualify only through their own age category. Low household income and the presence of a young child do not make the child's mother or caretaker independently eligible."
us,scenario_008,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_008,state_income_tax_before_refundable_credits,44,llm_error,taxable_income_or_deductions,False,,"The correct NJ calculation is gross income of $30,915 minus exactly $12,000 in exemptions: $1,000 for each spouse, $1,500 for each of the 6 dependents, and one $1,000 blind-or-disabled exemption for the spouse, even though the spouse is both blind and disabled. That leaves $18,915 of taxable income, all in the 1.4% bracket, for $264.81. Nothing else reduces this figure. The 18%-of-rent property-tax deduction is not taken, OTC health expenses produce no medical deduction, and the $50 tenant property tax credit and NJ child tax credit are refundable, so they sit outside this output. Models missed the answer in four main ways: they counted the blind and disabled exemptions twice, used the wrong exemption amounts, took deductions the reference does not allow, or subtracted refundable credits."
-us,scenario_008,state_refundable_credits,51,llm_error,state_local_rule,False,,"The reference is the sum of three NJ credits. The first is the NJ EITC: 40% of the full $8,231 federal 3+-child maximum, or $3,292.40, because earned income of about $30,500 sits below the MFJ phase-out start. The second is the NJ Child Tax Credit for the two children aged 5 and 1 under the schedule enacted June 30, 2026: $1,250 each, or $2,500, at NJ taxable income well under $30,000 after exemptions. The third is the $50 renter property tax credit. Even the closest models used the earlier $1,000-per-child CTC schedule. Many also chose the CTC tier from gross income (the $30k–$40k, $800 tier) instead of NJ taxable income, left out the $50 property tax credit, phased down or misstated the federal EITC maximum, or wrongly claimed NJ has no refundable credits. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned reference_later_law; adjudicated llm_error (state_local_rule). The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied New Jersey's pre-2026 child tax credit schedule. The reference applies P.L.2026, c.26, approved June 30, 2026, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors."
+us,scenario_008,state_income_tax_before_refundable_credits,45,llm_error,taxable_income_or_deductions,False,,"The reference has no adjustments to New Jersey gross income: $26,800 wages + $4,000 self-employment + $115 interest = $30,915. Exemptions total exactly $12,000: $1,000 each for the two spouses, $1,500 for each of the 6 dependents, and a single $1,000 blind-or-disabled exemption for the spouse. That leaves $18,915 of taxable income, and 1.4% of it is $264.81. The models went wrong in four main ways: counting blind and disabled as two separate exemptions, or using $1,500 for each spouse; leaving out the blind/disabled exemption or using federal AGI net of half the self-employment tax; adding a renter property-tax deduction or an OTC-medical deduction, neither of which appears in the reference; and subtracting the $50 NJ property tax credit or the NJ child tax credit, which are both refundable and therefore don't reduce tax before refundable credits. Several models also zeroed out the tax by citing standard deductions, thresholds or exemption reductions that New Jersey doesn't have."
+us,scenario_008,state_refundable_credits,52,llm_error,state_local_rule,False,,"The reference adds three New Jersey refundable credits. The first is the NJ EITC at 40% of the full $8,231 federal EITC ($3,292.40); earned income of about $30,500 is below the 2026 joint-filer phaseout start for 3+ children, so no phaseout applies. The second is the NJ child tax credit of $2,500 for the two children under 6, under the schedule New Jersey enacted on June 30, 2026, which replaced the old $1,000-per-child top tier; the tier is set by NJ taxable income, which falls well under $30,000 after exemptions for six dependents and a blind/disabled spouse. The third is the $50 property tax credit for this renting household. Even the best answers ($5,342.40) used the old $2,000 CTC schedule. Most other models also dropped the $50 renter credit, used the gross-income $800 tier, phased down the federal EITC, or left out the CTC entirely. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned reference_later_law; adjudicated llm_error (state_local_rule). The judge labeled the case, and the rows of claude-opus-5.5 and gpt-6.1-sol, reference_later_law because those models applied New Jersey's pre-2026 child tax credit schedule. The reference applies P.L.2026, c.26, approved June 30, 2026, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors."
us,scenario_008,tanf,1,llm_error,thresholds_rates,False,,"New Jersey's WorkFirst NJ TANF is not a residual top-up: it is gated on countable income falling below the family-size eligibility standard tied to the maximum grant (roughly $770/month for a household of eight) and on countable resources at or below the $2,000 limit. This household's $26,800 in wages plus $4,000 of self-employment income is about $2,576/month, which clears the standard by a wide margin even after New Jersey's 50% earned-income disregard, and the $69,500 bank balance independently fails the resource test. Six children plus a blind and disabled spouse make the household read as categorically needy, which is the trap; family size raises the grant schedule but nowhere near enough to admit income at this level. The grant is therefore $0."
us,scenario_009,federal_income_tax_before_refundable_credits,36,llm_error,thresholds_rates,False,,"The correct AGI is $105,900. It is made up of $60,000 in gross wages, counted in full because the listed employer-sponsored insurance premiums are not subtracted from wages, plus $20,400 of pension income and $25,500 of Social Security, which is 85% of $30,000. Since OBBBA made TCJA permanent, 2026 uses the $32,200 MFJ standard deduction, with no personal exemptions, and the 10% bracket ending at $24,800. That gives tax of $2,480 + 12% × $48,900 = $8,348. Most models got AGI right but then went wrong in one of four ways: they used guessed or 2025 pre-OBBBA deduction and bracket values, assumed the TCJA sunset (exemptions and a 15% bracket), subtracted the ESI premiums from wages, or submitted a number that contradicted their own correct work. Developer adjudication (2026-10-14): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The judge labeled eight models' rows reference_later_law (deepseek-v4-pro-0813, deepseek-v4.1-flash, gemini-3.7-flash, gemini-4-argon, grok-4.5, grok-4.6, grok-4.7 and grok-build-0.1) because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, personal exemptions and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, and the benchmark scores law published before the freeze; the rows are model errors, as the 2026-09-29 wave decided for the same rule in scenario_007's federal income tax."
us,scenario_009,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -99,7 +99,7 @@ us,scenario_012,head_wic_eligible,2,llm_error,categorical_eligibility,False,,"WI
us,scenario_012,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_012,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_012,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier in this output. At a school-meal FPG ratio of 0.81 and with categorical eligibility, the household is classified as FREE, so it receives no reduced-price support."
-us,scenario_012,snap,50,llm_error,taxable_income_or_deductions,False,,"The correct calculation uses only two deductions: the FY2026 $209 standard deduction and the 20% earned income deduction. That leaves net income of $1,258. The $450 rent is below half of that, and the household gets no utility allowance because it reports no heating, cooling or other utility expense and the engine does not read the $411 energy subsidy when it sets the allowance, so there is no excess shelter deduction. The 30% expected contribution is rounded up to $378, and $785 − $378 = $407 a month. Most models went wrong in one of three ways: they added an SUA because of the energy subsidy, they counted the unborn child as a fourth member, or they used FY2025 parameters ($768/$204) or skipped the rounding (giving $407.70 → $4,892)."
+us,scenario_012,snap,51,llm_error,taxable_income_or_deductions,False,,"The SNAP unit has 3 members. Net income is $1,833.33 minus the $209 standard deduction minus the $366.67 earned income deduction, rounded to $1,258. There is no shelter deduction: $450 rent is below half of net income, and under the 2026 rules the $411 energy subsidy does not give this household (no elderly or disabled members) a standard utility allowance. The 30% contribution, rounded up, is $378, so the benefit is $785 − $378 = $407 a month, or $4,884 a year. Most models went wrong in one of four ways: they added a LIHEAP-triggered utility allowance that inflated the shelter deduction, counted the unborn child as a fourth member, used FY2025 parameters ($768 maximum allotment, $204 standard deduction), or skipped SNAP's rounding steps and got $407.70 or $408 a month instead of $407."
us,scenario_012,spouse_chip_eligible,7,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the person not already qualify for Medicaid. At age 18, the spouse is Medicaid-eligible under Mississippi's OLDER_CHILD category, so that Medicaid eligibility precludes CHIP regardless of pregnancy status or how far household income falls below a CHIP-related threshold."
us,scenario_012,spouse_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Mississippi Medicaid applies an OLDER_CHILD eligibility pathway to the 18-year-old spouse. The spouse's MAGI of 0.67 times FPL is below the applicable income threshold, so this category produces eligibility despite the spouse not being a tax-unit dependent."
us,scenario_012,spouse_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC covers pregnant individuals who satisfy the program's income test. The spouse is pregnant, and the household's approximately $22,000 annual income is below the applicable 185% federal-poverty income limit, so the spouse meets both categorical and financial eligibility requirements."
@@ -107,21 +107,21 @@ us,scenario_012,ssi,1,parse_contract_failure,missing_output,False,,All wrong res
us,scenario_012,state_income_tax_before_refundable_credits,10,llm_error,thresholds_rates,False,,"Mississippi does not apply a single rate to every dollar of taxable income. The first $10,000 of taxable income is taxed at 0%, and the 2026 rate of 4% applies only to the amount above $10,000. On a joint return, $22,000 minus the $4,600 MFJ standard deduction, the $12,000 MFJ exemption and the $1,500 dependent exemption leaves $3,900 of taxable income, which falls entirely in the 0% band, so the tax is $0. That $0 joint result is lower than the $118 from the individual (combined-separate) computation, so it is the one used. Every model either taxed its taxable income from the first dollar or got the deductions and exemptions wrong, and none applied the $10,000 zero-rate band."
us,scenario_013,federal_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"The trap is the Social Security provisional-income test: half of the $23,736 benefit ($11,868) plus the $6,720 pension, $16 of dividends and $72 of capital gain gives provisional income of $18,676, below the $25,000 first base amount for a single filer, so none of the Social Security is includible. AGI is therefore $6,808, which is under the single standard deduction before any age-65 additional amount or the OBBBA senior deduction is even reached, leaving taxable income and tax of $0. A second, independent guardrail is that federal income tax before refundable credits is floored at zero — nonrefundable credits can only zero out liability, never drive it negative."
us,scenario_013,head_medicaid_eligible,3,llm_error,age_disability,False,,"The head is 80, so the 138% FPL MAGI adult expansion group (ages 19-64) is closed to them. The only route left is the aged/blind/disabled pathway, which counts Social Security in full. Social Security of $23,736 plus the $6,720 pension and $88 of investment income comes to about $30,544, or 1.91x FPL. That is far above the SSI benefit rate and the 100% FPL aged/disabled standard, and with SSI at 0 there is no SSI-linked eligibility either, so the engine returns category NONE. All three models answered Yes. One asserted eligibility without testing income, one excluded most of the Social Security, and one used AGI, which leaves out the nontaxable Social Security."
-us,scenario_013,snap,51,llm_error,categorical_eligibility,False,,"The trap is Arizona's broad-based categorical eligibility (BBCE). From March 2026 it covers households with gross income up to 200% of the poverty guideline ($2,608.33 a month, then $2,660 from October). This household's countable gross income is $2,539.33 a month, which counts Social Security, pension and dividends but not the capital gains. Because it is under that limit, the household is treated as meeting the net income and asset tests. The $298 maximum less 30% of net income comes out negative, but an eligible one-person household gets the $24 minimum allotment. Ten months at $24 gives $240. Every model stopped at the regular asset test, the net income test or the benefit formula and answered $0, missing the BBCE route and the minimum benefit."
+us,scenario_013,snap,52,llm_error,categorical_eligibility,False,,"From March 2026, Arizona's broad-based categorical eligibility (BBCE) covers households with gross income up to 200% of the poverty guideline. That limit is $2,608.33 a month, rising to $2,660 from October. This household's SNAP gross income is $2,539.33 a month, so the asset and net income tests count as met. Because it is a one-person categorically eligible household, it gets the $24 minimum allotment even though the formula ($298 minus 30% of net income) comes out negative. That gives $24 × 10 months = $240. Every model answered $0: most applied the ordinary elderly/disabled asset test or net income test, some used an old 185% BBCE limit, and the rest let the zero benefit formula override the one-person minimum allotment."
us,scenario_013,state_income_tax_before_refundable_credits,1,llm_error,taxable_income_or_deductions,False,,"Arizona’s standard deduction eliminates the remaining taxable income after excluding Social Security, so applying the 2.5% rate directly to gross dividends and capital gains is incorrect. The resulting Arizona income tax before refundable credits is zero."
us,scenario_013,state_refundable_credits,41,llm_error,state_local_rule,False,,"Arizona's Increased Excise Tax Credit (A.R.S. §43-1072.01) is a refundable credit of $25 for each taxpayer and dependent, capped at $100. A single filer qualifies when federal AGI is at or below $12,500. The credit does not require earned income, dependents, rent, property tax, or any tax liability. This head's AGI is $6,808 because Social Security is untaxed at this income level, so they qualify for $25 as a one-person tax unit. Almost every model either did not know this credit exists, mixed it up with the property tax credit, family credit, or EITC, or used the wrong income test or per-person amount."
us,scenario_014,federal_income_tax_before_refundable_credits,37,llm_error,thresholds_rates,False,,"The reference counts only the spouse's $88,927.65 in wages as income, because all VA benefits are excluded and the employer-sponsored insurance premium input does not reduce the listed gross wages. It then applies the 2026 OBBBA married-filing-jointly parameters: a $32,200 standard deduction, 10% up to $24,800 and 12% up to $100,800. That gives taxable income of $56,727.65 and tax of $2,480 + 12% × $31,927.65 = $6,311.32, with no nonrefundable credits because there are no dependents. The wrong answers come from four mistakes: assuming the TCJA sunset (personal exemptions and a 15% bracket), using stale 2025 or guessed deduction and bracket figures, subtracting the $14,717 ESI premium from wages, and submitting a final number that differs from the model's own correct or nearly correct arithmetic."
us,scenario_014,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"The tax unit is a married couple with no dependents, so the refundable Child Tax Credit cannot apply because there is no qualifying child. The childless EITC phases out completely at roughly $26,000 of earned income/AGI for joint filers in 2026, and the spouse's $88,928 in wages is far above that. Veterans benefits are nontaxable and do not create eligibility for any credit. There are also no education expenses (so no refundable AOTC) and no recovery rebate, so federal refundable credits are $0."
us,scenario_014,head_medicaid_eligible,1,llm_error,taxable_income_or_deductions,False,,"Medicaid eligibility for the head uses household tax-unit MAGI rather than only the head's earned income. The household's MAGI is approximately $183,584, or 4.11 times FPL, exceeding every applicable West Virginia Medicaid income limit, and the head has no alternative categorical pathway."
us,scenario_014,payroll_tax,16,llm_error,payroll_tax_base,False,,"Only the spouse has wages, so the correct answer is the 7.65% employee FICA rate applied to her full $88,928 gross wages: $5,513.51 Social Security plus $1,289.45 Medicare, or $6,802.96. The facts do not say she pays premiums through a salary-reduction (Section 125) plan, so the $14,717 in employer-sponsored insurance premiums does not come out of the FICA wage base. Veterans benefits are not wages, and West Virginia has no employee state payroll tax. The largest group of wrong answers (7 models) assumed a cafeteria plan that the facts never mention and taxed only $74,211. Most of the rest applied 7.65% to $88,928 and then invented an extra tax, made an arithmetic slip, or submitted a number different from their own total."
-us,scenario_014,state_income_tax_before_refundable_credits,51,llm_error,thresholds_rates,False,,"West Virginia taxable income is federal AGI of $88,927.65 (the spouse's wages; veterans benefits are excluded, and the listed ESI premiums are not subtracted) minus two $2,000 personal exemptions, which gives $84,927.65. WV has no standard deduction. The 2026 MFJ schedule is 2.11% up to $10k, 2.81% to $25k, 3.16% to $40k, 4.22% to $60k and 4.58% above that, which yields 211 + 421.50 + 474 + 844 + 1,141.69 = $3,092.19 with no nonrefundable credits. Most models got the $84,928 base right but used the 2025 schedule (2.22%–4.82%, giving $3,255.03) or the 2024 schedule (2.36%–5.12%, giving $3,459.81). A second group subtracted the $14,717 ESI premium, invented standard deductions or phantom credits, or applied flat rates."
+us,scenario_014,state_income_tax_before_refundable_credits,52,llm_error,thresholds_rates,False,,"The WV base is simple. AGI is the spouse's $88,927.65 in wages, since veterans benefits are excluded and the ESI premiums are not subtracted. WV has no standard deduction, so subtracting two $2,000 exemptions gives $84,927.65 of taxable income. The trap is the rate schedule: PolicyEngine taxes this under the further-reduced 2026 joint brackets of 2.11%/2.81%/3.16%/4.22%/4.58%. That gives 211 + 421.50 + 474 + 844 + 4.58% × 24,927.65 = $3,092.19, and no nonrefundable credits apply. Most models had the right base but used the 2025 schedule (2.22%–4.82%, giving $3,255.03) or the 2024 schedule (2.36%–5.12%, giving $3,459.81). Others shrank the base by subtracting the ESI premiums, inventing standard deductions, or applying credits that don't exist."
us,scenario_015,federal_income_tax_before_refundable_credits,38,llm_error,thresholds_rates,False,,"The correct path is AGI of about $42,909: $45,000 wages, minus about $1,042 in traditional 401(k) deferrals, plus $2,000 interest, minus the $3,000 capital-loss cap, minus the $49 IRA deduction. Workers' comp is excluded, and child support, health premiums, and Roth contributions are not deductible. Subtracting the 2026 single standard deduction of $16,100 (OBBBA made the TCJA structure permanent) leaves taxable income of $26,809. Tax is 10% up to $12,400 and 12% above that, which gives $2,969 with no nonrefundable credits. Most models went wrong on the parameters: they used the 2024/2025 standard deduction and bracket thresholds ($14,600, $15,000, $15,400 or $15,750; $11,600, $11,925 or $12,150), or assumed the TCJA had expired, bringing back personal exemptions and a 15% bracket. Others subtracted health premiums, made up mortgage interest to itemize, invented credits, or made arithmetic errors after getting the right taxable income."
us,scenario_015,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"The head files alone as a single adult with no qualifying children and earns $45,000 in wages, so no refundable credit applies. The refundable CTC/ACTC requires a qualifying child. The childless EITC phases out completely at roughly $19–20k of earned income/AGI in 2026. There are no education expenses, so the refundable AOTC is $0, and the Recovery Rebate and payroll tax credits are inactive in 2026. Any positive figure means the model either invented a child-based credit or applied the with-children EITC schedule."
us,scenario_015,head_chip_eligible,1,parse_contract_failure,other,False,,"CHIP eligibility runs through categorical gates first: coverage is for children under 19 and, at state option, pregnant women, so a 36-year-old head never reaches the income test — PolicyEngine assigns chip_category = 3 (categorically ineligible) and is_chip_eligible = False. The ~$43,450 household income and Indiana's Hoosier Healthwise/CHIP income thresholds are irrelevant to a person who fails the age/pregnancy gate, and the separate Medicaid pathways also return NONE for a non-aged, childless adult at that income in a state whose adult expansion he clears on neither disability-SSI nor MAGI grounds. The single wrong answer stated this categorical rule correctly in prose and then emitted the opposite binary."
us,scenario_015,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The head is a 36-year-old non-dependent adult with MAGI at 2.69x FPL. That is far above Indiana's 138% FPL expansion-adult limit, so the MAGI pathway fails. The head receives no SSI, and PolicyEngine's aged/blind/disabled pathway applies SSI-style income limits plus a small resource cap. Roughly $45,000 of earnings and about $585,000 in bank and stock assets fail both tests, so medicaid_category is NONE. Both models treated the disability flag by itself as qualifying, even though disability opens a pathway only when its income and asset tests are met, and PolicyEngine does not model a working-disabled buy-in with relaxed limits."
us,scenario_015,head_medicare_eligible,4,llm_error,age_disability,False,,"A person under 65 becomes eligible for Medicare on disability grounds only after 24 months of entitlement to Social Security Disability Insurance, or through the ESRD or ALS pathways. Being disabled is not enough on its own. The head is 36, has no Social Security disability benefits, and has no ESRD or ALS status. That means they don't meet the age-65 test or any disability-entitlement pathway, so they are not Medicare eligible. All four models treated the bare 'is disabled' flag as automatic Medicare eligibility and skipped the SSDI-receipt requirement."
-us,scenario_015,local_income_tax,51,prompt_ambiguity,state_local_rule,False,,"PolicyEngine's local_income_tax includes Indiana county income tax. The household gave no county, so the engine assigned Adams County and applied its 1.6% rate to Indiana AGI of $41,909.38. That AGI is federal AGI of $42,909.38 (after the $1,041.93 401(k) exclusion, the $3,000 capital-loss deduction and the $48.69 IRA deduction) minus the $1,000 Indiana exemption, which gives $670.55. The prompt's definition of the output lists only NYC, Philadelphia, Kansas City and St. Louis taxes. It can be read two ways: (1) the list is exhaustive, so Indiana county tax is out of scope and the answer is $0; or (2) the list gives examples ('where applicable') of local taxes in the output, which also covers Indiana county tax. The prompt also never names a county. Almost every model took reading (1), and a few wrongly claimed Indiana has no local income tax at all. Developer adjudication (2026-10-10): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state. Reference depends on an unlisted input; output excluded from scoring (county of residence (Indiana county income tax))."
-us,scenario_015,payroll_tax,9,llm_error,payroll_tax_base,False,,"Employee FICA applies to the full $45,000 of gross wages: 6.2% Social Security ($2,790, well under the 2026 wage base) plus 1.45% Medicare ($652.50) gives $3,442.50. Indiana has no mandatory employee state payroll tax, and wages are far below the $200,000 Additional Medicare Tax threshold. The most common wrong move was shrinking the FICA base. Four models treated the listed $2,080 health insurance premiums as a Section 125 pre-tax salary reduction, and one subtracted the traditional 401(k) deferral, even though elective 401(k) deferrals stay subject to FICA. The rest reached or approached $3,442.50 and then broke the total with arithmetic slips or made-up add-ons."
+us,scenario_015,local_income_tax,52,prompt_ambiguity,state_local_rule,False,,"The prompt defines local_income_tax as the separate local-tax output covering ""NYC income tax, Philadelphia wage tax, Kansas City earnings tax, and St. Louis earnings tax where applicable."" That list reads as complete, but PolicyEngine's local_income_tax also includes Indiana county income tax. Because the prompt gives no county, the engine defaults to Adams County and applies its 1.6% rate to Indiana AGI of $41,909.38 (federal AGI of $42,909.38 minus the $1,000 personal exemption), which gives $670.55. The two readings are (a) the list is complete, so an Indiana household owes $0, or (b) the list only gives examples, so Indiana county tax counts and a default county applies. Every model took reading (a), and a few also wrongly said Indiana has no local income tax at all. Developer adjudication (2026-10-10): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state. Reference depends on an unlisted input; output excluded from scoring (county of residence (Indiana county income tax))."
+us,scenario_015,payroll_tax,10,llm_error,payroll_tax_base,False,,"The employee payroll tax base here is the full $45,000 of gross wages. Traditional 401(k) elective deferrals stay in FICA wages under IRC §3121(v)(1)(A). The listed health insurance premiums are ordinary premiums, not a Section 125 salary reduction. That gives 6.2% × $45,000 = $2,790 plus 1.45% × $45,000 = $652.50, for $3,442.50, with no Additional Medicare Tax and no Indiana employee payroll tax. The wrong answers come from three sources: shrinking the FICA base by the 401(k) deferral or the health premiums, adding a nonexistent state or county component, or arithmetic and transcription slips after the right method."
us,scenario_015,state_income_tax_before_refundable_credits,34,llm_error,thresholds_rates,False,,"Indiana tax here is federal AGI of $42,909.38 (wages minus the $1,042 traditional 401(k), plus $2,000 interest, minus the $3,000 capital-loss cap, minus the $49 IRA deduction; workers' compensation excluded), less the single $1,000 personal exemption, giving $41,909.38. That amount is taxed at Indiana's 2026 flat rate of 2.95%, which gives $1,236.33. Most models used the wrong 2026 rate (3.0%, 3.05%, 2.85%, 2.90%, 2.97%, 3.025%). Others made up deductions that do not apply: a disability exemption for a taxpayer under 65, a health-premium deduction, a child-support deduction, or a 'standard' deduction. A few started from the wrong AGI or from federal taxable income."
us,scenario_015,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_016,federal_income_tax_before_refundable_credits,8,llm_error,taxable_income_or_deductions,False,,"The household reports $47,000 of deductible medical outlays — $26,000 of health insurance premiums excluding Medicare Part B, $20,000 of other medical expenses, and $1,000 of over-the-counter health expenses — against an AGI of $32,137.94 (wages of $34,638 less the $2,500 statutory cap on the student loan interest deduction). Net of the 7.5%-of-AGI floor of $2,410, the itemized medical deduction of roughly $44,590 exceeds AGI outright, so itemizing dominates the ~$16,100 standard deduction, taxable income is zero, and federal income tax before refundable credits is $0. Seven of the eight models ran only the standard-deduction path and clustered in the $1,271–$1,873 band; the eighth recognized the medical deduction but reported the unused excess as a large negative liability instead of flooring the tax at zero."
@@ -140,13 +140,13 @@ us,scenario_016,ssi,1,parse_contract_failure,missing_output,False,,All wrong res
us,scenario_016,state_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_016,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_016,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_018,federal_income_tax_before_refundable_credits,42,llm_error,taxable_income_or_deductions,False,,"The reference is a plain calculation. AGI is $61,592.09 (wages minus the traditional 401(k) and IRA contributions, plus $50 of interest). Subtracting the 2026 OBBBA standard deduction of $16,100 leaves taxable income of $45,492.09. Tax at 10% up to $12,400 and 12% above that is $5,211.05, with no OBBBA overtime or car-loan interest deduction, no itemizing, and no credits. The wrong answers fall into four groups: (1) models that added a $6,760 overtime deduction and/or a $1,625 car-loan interest deduction that the facts don't support (no FLSA overtime premium is listed, and the engine's overtime deduction counts only that premium; nothing says the vehicle qualifies); (2) models that used an old or guessed standard deduction or bracket edge, or assumed the TCJA expired; (3) models that itemized after counting the same $9,000 premium twice and including over-the-counter costs; (4) models whose arithmetic was inconsistent or who gave no answer."
+us,scenario_018,federal_income_tax_before_refundable_credits,43,llm_error,taxable_income_or_deductions,False,,"The reference starts from AGI of $61,592.09 (wages minus traditional 401(k), minus the IRA deduction, plus $50 interest). It subtracts only the 2026 OBBBA single standard deduction of $16,100, giving $45,492.09 of taxable income. The tax is 10% on the first $12,400 plus 12% on the rest, or $5,211.05. The models went wrong in four main ways: (1) claiming a qualified overtime deduction or a car loan interest deduction that the facts don't support (wages of $68,005 ≈ $26 × 50 × 52 show no time-and-a-half pay, and nothing says the loan is on a qualifying new vehicle); (2) using 2024/2025 or made-up 2026 standard deductions and bracket edges; (3) assuming the TCJA expired in 2026 (OBBBA made it permanent); (4) itemizing by counting the same $9,000 premium twice. A few models also invented credits or made arithmetic slips."
us,scenario_018,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_018,payroll_tax,7,llm_error,payroll_tax_base,False,,"Employee FICA applies to the full $68,005 of wages here: $4,216.31 of Social Security tax plus $986.07 of Medicare tax equals $5,202.38. The listed health-insurance premiums were not identified as employee pre-tax salary-reduction contributions, and Arizona's lack of a separate state payroll tax does not remove federal employee FICA."
us,scenario_018,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_018,state_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"Most models got Arizona AGI right at about $61,592 and knew the flat 2.5% rate. What separates them is the 2026 Arizona single standard deduction: it is $16,100, which is the $15,750 base set by HB 4168, indexed the same way as the federal basic standard deduction. That leaves taxable income of $45,492.09 and tax of $1,137.30. Most wrong answers used a stale or guessed deduction ($13,790–$15,750). Others added a personal exemption that does not exist, itemized medical costs above PolicyEngine's $10,637.30 itemized total, cut AGI by health premiums, used repealed progressive brackets, or submitted a number that contradicted their own arithmetic."
us,scenario_018,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_020,federal_income_tax_before_refundable_credits,51,prompt_ambiguity,other,False,,"This Texas filer has no state income tax, so the SALT itemized deduction uses the general sales tax election. The frozen reference adds a sales tax amount of about $3,256, projected from the IRS's 2023 optional table, to the $13,834 of real estate taxes. That gives $17,090.34 of itemized deductions, so it itemizes and the $196 gift falls below the 0.5%-of-AGI floor. The release's sales tax convention (c_irs_sales_tax_2025) uses the IRS 2025 table, whose $1,595 for a single Texas filer at this income leaves SALT below the $16,100 standard deduction. Under it the filer takes the standard deduction plus the $196 non-itemizer charitable deduction, and policyengine-us 2.60.0 gives $68,334.73 with NIIT. The rest of the reference follows current law after the One Big Beautiful Bill Act (OBBBA): AGI is $300,150.19 after the above-the-line alimony deduction, the full $300,000 of wages is counted with no subtraction for health premiums, unreimbursed employee expenses stay permanently non-deductible, the overtime deduction is fully phased out, and the $222.64 net investment income tax (NIIT) is added. The models that got closest took the standard deduction, as the convention does; those that also took the $196 non-itemizer charitable deduction land within $0.14 of $68,334.73 with NIIT or of $68,112.09 without it. The rest dropped alimony, subtracted the health premiums, assumed the TCJA would expire, or left out NIIT. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (other). The frozen reference is 68,056.71; the alternative is 68,112.09. The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it. Reference depends on an unlisted input; output excluded from scoring (whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say)."
+us,scenario_020,federal_income_tax_before_refundable_credits,52,prompt_ambiguity,other,False,,"Texas has no income tax, so the SALT deduction for this filer is the $13,834 of real estate taxes plus a general sales tax amount from the IRS optional sales tax table. That brings SALT to $17,090.34, well under the 2026 cap of $40,400 and above the $16,100 standard deduction, so the filer itemizes. Itemizing replaces the $196 non-itemizer charitable deduction with $0 because the gift falls below the 0.5%-of-AGI floor. The correct path also keeps the $5,709 alimony deduction (AGI $300,150), adds the $222.64 NIIT, and leaves out ESI premiums, overtime, unreimbursed employee expenses and pre-TCJA sunset rules. The best answers ($68,334.59 and $68,403) took the standard deduction because they missed the sales-tax add-on. Most other answers dropped the alimony deduction, left out the NIIT, or applied TCJA-sunset rules that OBBBA displaced. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (other). The frozen reference is 68,056.71; the alternative is 68,112.09. The prompt defines the output as federal individual income tax after nonrefundable credits and before refundable credits. The reference adds the net investment income tax (26 U.S.C. 1411), which Form 1040 reports on Schedule 2, Part II with self-employment tax, after line 22's tax after nonrefundable credits; read as line 22, the output excludes it. Reference depends on an unlisted input; output excluded from scoring (whether federal income tax before refundable credits includes the net investment income tax, which the output's definition does not say)."
us,scenario_020,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_020,payroll_tax,33,llm_error,thresholds_rates,False,,"The reference is plain FICA arithmetic on the stated $300,000 gross wages: 6.2% Social Security on the 2026 wage base of $184,500 ($11,439), plus 1.45% Medicare on all $300,000 ($4,350), plus 0.9% Additional Medicare Tax on the $100,000 above the $200,000 single threshold ($900). That totals $16,689. Most models missed it in one of three ways. Some used a stale or guessed wage base, such as the 2025 figure of $176,100 or the 2024 figure of $168,600. Some cut FICA wages by the $8,389 ESI premium even though the facts never call it a pre-tax salary reduction. A few added the $42,857 FLSA overtime premium on top of wages that already include it. Several models also derived $16,689 and then submitted a different, unrelated number."
us,scenario_020,state_income_tax_before_refundable_credits,1,llm_error,state_local_rule,False,,"Texas imposes no individual state income tax, so a Texas resident's state income tax before refundable credits is zero. The nationwide aggregation does not create liability in jurisdictions where the household does not reside."
@@ -158,15 +158,15 @@ us,scenario_021,self_employment_tax,1,parse_contract_failure,missing_output,Fals
us,scenario_021,snap,1,llm_error,thresholds_rates,False,,"An elderly or disabled SNAP household is exempt from the gross-income test but must still satisfy the one-person net-income limit. With monthly net income of approximately $2,063 even under the model’s own deductions, the household fails that test, so the calculation stops at ineligibility and produces $0 rather than an allotment."
us,scenario_021,state_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"Missouri tax is computed from Missouri taxable income after applicable deductions and the 2026 taxable-income threshold, not by applying a flat rate directly to taxable pension income. After those steps, this household has no Missouri taxable income and therefore owes $0 before refundable credits."
us,scenario_021,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_022,federal_income_tax_before_refundable_credits,51,prompt_ambiguity,taxable_income_or_deductions,False,,"With no dependent child, the 77-year-old widow does not qualify as a qualifying surviving spouse, so the return is taxed on the single schedule. Itemizing beats the $18,150 standard deduction. The prompt lists no state income tax withheld or paid. In the reference, SALT is $14,887.85 under OBBBA's $40,400 cap: $8,829 of property tax plus $6,058.85, the engine's formula estimate of California income tax withheld on federal AGI. Charity is $14,847.67 after the new 0.5%-of-AGI floor. Add the $910 QBI deduction on rental income and the senior deduction phased down to $3,951.05, and taxable income is $74,552.59, which gives $11,113.57 at 2026 single rates. The wrong answers split three ways: some used joint/surviving-spouse brackets or the standard deduction, some assumed the TCJA would sunset (personal exemptions, 2% miscellaneous deductions, 15%/25% brackets) even though OBBBA made it permanent, and some filed single but put no state income tax or their own California tax estimate in SALT, an input the prompt leaves unlisted, or dropped the senior or QBI deduction. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 11,113.57; the alternative is 11,895.23. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of California withholding on federal AGI ($6,058.85), while read as the household paying its 2026 California income tax during the year the deduction takes the engine's liability ($2,505.87). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)."
+us,scenario_022,federal_income_tax_before_refundable_credits,52,prompt_ambiguity,taxable_income_or_deductions,False,,"This 77-year-old surviving spouse has no dependent child, so they cannot use qualifying-surviving-spouse status and must file single. AGI is $109,149 (85% of Social Security is taxable). The correct path is to itemize $29,736: SALT of $14,888, which is property tax plus about $6,059 of estimated CA income tax and sits under the 2026 OBBBA cap of $40,400, plus charity of $15,393 less the 2026 floor of 0.5% of AGI. Then subtract the senior deduction, phased down to $3,951, and the $910 QBI deduction on rental income, giving taxable income of $74,553 taxed at 2026 single brackets (10% to $12,400, 12% to $50,400, 22% above). Most models went wrong in one of three ways: using joint/QSS brackets and the QSS standard deduction, leaving CA income tax out of SALT or understating it and skipping the senior or QBI deduction, or applying pre-TCJA law with personal exemptions, 2%-floor miscellaneous deductions and 15%/25% brackets. OBBBA made the TCJA rules permanent, so that pre-TCJA treatment does not apply. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 11,113.57; the alternative is 11,895.23. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of California withholding on federal AGI ($6,058.85), while read as the household paying its 2026 California income tax during the year the deduction takes the engine's liability ($2,505.87). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)."
us,scenario_022,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_022,head_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Age 77 establishes an aged status but does not itself confer Medicaid eligibility, and Medicare eligibility or assumed enrollment is not a Medicaid eligibility pathway. The head receives no SSI and qualifies under none of California's Medicaid categories, so the aged status cannot overcome the absence of a qualifying pathway."
us,scenario_022,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_022,payroll_tax,43,llm_error,state_local_rule,False,,"The federal part is simple: 6.2% Social Security ($3,263.37) plus 1.45% Medicare ($763.21) on $52,635 of wages, with no Additional Medicare Tax. The trap is California's mandatory employee State Disability Insurance (SDI) contribution. For 2026 it is 1.3% of all wages with no cap, which is $684.26, for a total of $4,710.83. Most models either left SDI out and stopped at $4,026.58, or used an earlier year's SDI rate: 1.2% for 2025 ($4,658.20), 1.1% for 2024 ($4,605.57), or 0.9% for 2023 ($4,500.30)."
us,scenario_022,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_022,state_income_tax_before_refundable_credits,51,llm_error,taxable_income_or_deductions,False,,"To reach the reference, three steps are needed. First, exclude Social Security, which gives CA AGI of $91,524. Second, itemize $30,390: property tax $8,829, charity $15,393 and unreimbursed employee expenses of $8,351 less a 2% floor based on federal AGI ($109,149), leaving about $6,168. California never adopted the TCJA suspension of miscellaneous itemized deductions, so this deduction still applies. Third, tax the resulting $61,134 on the single schedule. The head has no dependent child, so they don't qualify as a surviving spouse and file single: $2,274.81 in tax, minus the $153 personal and $153 senior exemption credits. The models went wrong in two main ways: (a) they used joint/qualifying-surviving-spouse brackets and joint exemption credits, or (b) they left out the employee-expense deduction or based its 2% floor on CA AGI instead of federal AGI. Smaller groups dropped the senior exemption credit, left Social Security in income, or used the standard deduction."
+us,scenario_022,state_income_tax_before_refundable_credits,52,llm_error,household_unit_or_filing_status,False,,"The head has no dependent, so being a surviving spouse does not qualify them for qualifying-surviving-spouse status. They file single: the single brackets apply, the standard deduction is $5,706, and they get one personal exemption credit plus one senior exemption credit, totalling $306. California does not follow TCJA's removal of miscellaneous itemized deductions, so $6,168 of unreimbursed employee expenses (the amount above the 2% floor) joins $8,829 of property tax and $15,393 of charity, for $30,390 in itemized deductions. California AGI excludes Social Security, so AGI is $91,524 and taxable income is $61,134. Single-bracket tax is $2,274.81, and subtracting the $306 of credits gives $1,968.81. The wrong answers mostly used joint/QSS brackets with doubled exemption credits, left out the miscellaneous deduction, or dropped the senior exemption credit."
us,scenario_022,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_023,federal_income_tax_before_refundable_credits,44,llm_error,credit_phaseout,False,,"AGI is about $22,534: wages minus the traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the IRA deduction. Subtracting the 2026 OBBBA single standard deduction of $16,100 leaves $6,434.34 of taxable income, all taxed at 10%, which gives $643.43. No nonrefundable credit reduces that tax. The Saver's Credit is $0 because Form 8880 subtracts distributions received during the testing period, and the $8,000 403(b) distribution exceeds all $3,599 of retirement contributions. The AOTC is $0 because no qualified tuition expenses are listed, and an unlisted amount counts as 0. The largest group of wrong answers reached $0 by applying a $1,000 Saver's Credit or an AOTC to wipe out the tax. Others subtracted the ESI premiums as pre-tax wages, dropped the distribution from income, or used a pre-OBBBA standard deduction, either the TCJA-sunset amount or the 2024/2025 amount."
+us,scenario_023,federal_income_tax_before_refundable_credits,45,llm_error,categorical_eligibility,False,,"AGI is $22,534 ($14,664 W-2 wages after the traditional 401(k) deferral, plus $8,000 of taxable 403(b) distributions, minus the $130 IRA deduction). The 2026 OBBBA single standard deduction of $16,100 leaves $6,434 taxable, so tax at 10% is $643.43. No nonrefundable credit applies. The Saver's Credit is zero because IRC §25B(d)(2) reduces eligible contributions by distributions received in the testing period, and the $8,000 403(b) distribution exceeds the $3,599 in total contributions. The AOTC is zero because no qualified tuition expense is listed. Most wrong models either claimed a $1,000 Saver's Credit or an AOTC to bring the tax to zero, or used a pre-OBBBA standard deduction (with or without a personal exemption)."
us,scenario_023,federal_refundable_credits,20,llm_error,credit_phaseout,False,,"This case has two traps, and a model had to avoid both to reach $0. First, the AOTC eligibility flags do not create a credit. The credit is 100% of the first $2,000 of qualified tuition expenses plus 25% of the next $2,000, and no expenses are listed, so the base is $0 and the refundable 40% portion is also $0. Second, the childless EITC phaseout is measured on the greater of earned income or AGI. AGI is about $22,535: wages of $17,443, minus the $2,778 traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the $130 IRA deduction. That is above the 2026 single childless completion point of about $19,540, so the EITC is fully phased out."
us,scenario_023,free_school_meals_eligible,3,llm_error,categorical_eligibility,False,,"The household qualifies for the FREE meal tier through California’s universal program and categorical eligibility, despite its 1.59 federal-poverty-guideline ratio exceeding the federal income cutoff. The requested output is narrower: it equals 1 only when PolicyEngine returns positive annual free-school-meal support, and this adult-only household has no student receiving meals, so the annual support is zero."
us,scenario_023,head_chip_eligible,2,llm_error,categorical_eligibility,False,,"CHIP eligibility requires being in a covered CHIP category and not already eligible for Medicaid; it is not a general adult coverage pathway based solely on falling below an extended income ceiling. The 28-year-old head qualifies for Medicaid through the WORKING_DISABLED_BUY_IN category, which places them in CHIP category 3 and categorically excludes them from CHIP."
@@ -177,28 +177,28 @@ us,scenario_023,local_income_tax,1,parse_contract_failure,missing_output,False,,
us,scenario_023,payroll_tax,45,llm_error,thresholds_rates,False,,"The payroll tax base is the full gross wage of $17,442.65. Section 125 premiums and 401(k) deferrals do not reduce it, and 403(b) distributions are not wages. On that base: Social Security at 6.2% is $1,081.44, Medicare at 1.45% is $252.92, and California's mandatory employee SDI at the 2026 rate of 1.3% with no wage cap is $226.75 (226.75 / 17,442.65 = 1.3%), for a total of $1,561.12. Models went wrong in one of three ways: they left out CA SDI entirely (FICA-only answers of about $1,334), they used an older SDI rate (1.1% for 2024, 1.2% for 2025, or other guesses), or they subtracted the $8,089 employer-sponsored insurance premium, or 401(k) deferrals, from the wage base."
us,scenario_023,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_023,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_023,snap,51,prompt_ambiguity,age_disability,False,,"For SNAP, a ""disabled member"" means someone who receives a disability-based benefit such as SSI or SSDI (7 U.S.C. 2012(j)). Being flagged ""is disabled"" is not enough. On the frozen reference's capped reading, there is no excess medical deduction and shelter is capped at $744. Gross income passes California's 200% FPL BBCE screen. The $876.51 pre-rounding net amount becomes $877 under the current engine, whose 30% expected contribution rounds up to $264. Under c_snap_hold_fy2026, the $298 maximum applies all 12 months, giving $34/mo ($408/year). The published $461.34 is the excluded frozen reference, including the engine's projected FY2027 increase for October–December. Most models applied uncapped shelter plus a medical deduction and reached the maximum or near-maximum allotment. Some $0 answers applied a 130% gross test, which BBCE replaces, or simply asserted the income was too high. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 3,576.00 with every publication convention and upstream fix (the frozen-engine value above was 3,596.04)."
-us,scenario_023,ssi,2,llm_error,categorical_eligibility,False,,"SSI requires the individual to satisfy the aged, blind, or disabled categorical gate before income exclusions or benefit-rate calculations apply. The head's unlisted SSI disability status is false under the prompt instructions, so `is_ssi_aged_blind_disabled=False` and SSI is $0 without an income calculation."
-us,scenario_023,state_income_tax_before_refundable_credits,51,llm_error,thresholds_rates,False,,"California AGI is $22,534 (wages less the $2,778 traditional 401(k) deferral, plus the $8,000 taxable 403(b) distribution, less the $130 IRA deduction). With the 2025 published parameters ($5,706 standard deduction, 1% bracket to $11,079, then 2%), tax is $225.78. Subtracting the $153 personal exemption credit and the $60 nonrefundable renter's credit leaves a small positive residual of $12.78. Models went wrong in four main ways: they assumed the credits wipe out the tax entirely, they projected 2026 indexing beyond the 2025 values, they dropped one of the two credits, or they left out the $8,000 distribution. Disability does not earn an extra California exemption credit; only blindness and age 65+ do."
-us,scenario_023,state_refundable_credits,50,llm_error,credit_phaseout,False,,"The only California refundable credit here is the childless CalEITC. It is available to filers aged 18 and older with no qualifying child, it does not depend on getting the federal EITC, and it phases out over income up to $32,901. In the phase-out range, the credit is reduced using the greater of earned income ($17,442.65) or AGI ($22,534.34, which includes the $8,000 taxable 403(b) distribution). That deep phase-out leaves $95.15. Models that answered $0 wrongly ruled out the credit: they claimed it requires a child, is nonrefundable, depends on the federal EITC, ends at a lower income, or is blocked by the investment-income limit. Most models that answered above about $110 ran the phase-out on wages alone or barely applied it. Others, such as claude-opus-5.5 and grok-4.7, ran it on AGI but used amounts that overstated the credit, and one model (qwen-3.7-max) added the nonrefundable renter's credit."
+us,scenario_023,snap,52,prompt_ambiguity,age_disability,False,,"The key point is that the generic ""is disabled"" flag does not make the head a SNAP-disabled member. Under the SNAP definition (7 U.S.C. 2012(j)), that status requires receiving disability benefits such as SSI or SSDI, and this head receives neither (SSI is $0 because income is too high). So the household is not an elderly/disabled household. It gets no excess medical deduction, and its excess shelter deduction is capped at the FY2026 limit of $744. Monthly net income is $2,120.25 gross − $290.72 (20% earned-income deduction) − $209 (standard deduction) − $744 (capped shelter) ≈ $876.51. That passes CA's 200% FPL BBCE gross screen and the 100% FPL net test, giving $298 − $262.80 = $35.20 a month for 9 months and $48.18 a month for the last 3 months (higher maximum allotment), or $461.34 in total. Most models gave uncapped shelter and medical deductions on the strength of the disability flag, which overstated the benefit. Models that answered zero either applied the 130% FPL gross test without CA's BBCE or never applied the capped shelter deduction. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 461.339722 to 3596.039795 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI itself stays $0 under either reading (the head's countable income exceeds the federal benefit rate); SNAP moves because SSI-disabled status switches on the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 3,576.00 with every publication convention and upstream fix (the frozen-engine value above was 3,596.04)."
+us,scenario_023,ssi,3,llm_error,categorical_eligibility,False,,"SSI has a categorical gate that comes before the income test. The person must count as aged (65+), blind, or disabled for SSI, and the engine sets is_ssi_aged_blind_disabled=False for this 28-year-old head. That makes SSI $0 regardless of income. Both models that gave numbers read the generic 'is disabled' flag as automatic SSI eligibility and went straight to the income-exclusion arithmetic. Each then made a second error that pushed its result above zero: one left out the $8,000 403(b) unearned income, and the other added California's state supplement (SSP) into the federal SSI output."
+us,scenario_023,state_income_tax_before_refundable_credits,52,llm_error,thresholds_rates,False,,"California AGI is $22,534: wages minus the 401(k) deferral, plus the $8,000 taxable 403(b) distribution, minus the IRA deduction. The $5,706 single standard deduction leaves taxable income of $16,828. The 2025 rate schedule (1% up to $11,079, then 2%) gives $225.78 of tax. Two nonrefundable credits reduce this: the $153 personal exemption credit and the $60 renter's credit, totalling $213 and leaving only $12.78. Because the remainder is so small, the errors split into three groups. Models that left out the 403(b) distribution, wrongly assumed the credits wiped out the tax, or added a nonexistent disability exemption credit reported $0. Models that left out one credit, or did not let California follow the federal treatment of the 401(k) and IRA, reported $72–$291. Models that projected 2026 inflation increases to the deduction and credit, instead of using the 2025 published amounts, came out a few dollars low."
+us,scenario_023,state_refundable_credits,51,llm_error,credit_phaseout,False,,"The only California refundable credit here is the childless CalEITC. It is available to filers age 18 and older without children and does not depend on federal EITC eligibility. Its phase-out applies to the greater of earned income ($17,442.65 in wages) or AGI ($22,534.34, which includes the $8,000 taxable 403(b) distribution net of 401(k) and IRA deductions). At AGI of about $22.5k, the credit sits in the slow final phase-out tail and comes to $95.15. Models that answered $0 wrongly ruled the filer ineligible or treated the credit as already phased out. Models with larger amounts phased out on earnings alone, used a single linear phase-out, or skipped the phase-out entirely."
us,scenario_023,tanf,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_025,federal_income_tax_before_refundable_credits,33,llm_error,thresholds_rates,False,,"AGI is $94,925.30 ($62,725.29 wages + $32,200 pension). The 2026 OBBBA married-filing-jointly standard deduction of $32,200 is far larger than the $2,354 in itemized deductions, so taxable income is $62,725.30. Under the 2026 brackets (10% up to $24,800, 12% up to $100,800), tax is $2,480 + 12% × $37,925.30 = $7,031.04, and no nonrefundable credits apply. Most wrong answers used 2025 or pre-OBBBA inflation-projected deduction and bracket amounts, or assumed the TCJA had expired (personal exemptions, a 15% bracket). Several models reached the right figure and then submitted a different number."
us,scenario_025,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_025,head_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility requires age 65 in the absence of a qualifying disability, ESRD, or other explicitly listed pathway. The 61-year-old head has no such condition, so the age test yields 0 (No)."
us,scenario_025,payroll_tax,8,llm_error,payroll_tax_base,False,,"The spouse's $62,725 of wages are the only FICA-covered earnings. They are taxed at 6.2% for Social Security and 1.45% for Medicare, which gives $4,798.48 ($3,888.97 + $909.52). The head has no wages. The $32,200 pension is outside the FICA base. The head's ESI premium is not subtracted from the spouse's wages. Ohio has no employee state payroll tax, and Additional Medicare Tax does not apply. Several models got this computation right but submitted a different number. Others added Ohio income tax, subtracted the ESI premium, used the combined employer-plus-employee rate, or counted wages for the head."
-us,scenario_025,state_income_tax_before_refundable_credits,51,llm_error,thresholds_rates,False,,"Ohio's 2026 schedule charges a $332 base amount plus 2.75% of taxable nonbusiness income above $26,050. Most models taxed only 2.75% of the excess and dropped the $332 base, which leaves them about $327 low. The second trap is the unreimbursed medical care deduction. Because the head can get an employer plan, the $6,500 of premiums and $800 of other medical costs both go through the 7.5%-of-FAGI floor, so the deduction is only $180.60 (not $0, not the full premium, and without the $21,208 employer premium or the $50 OTC spending). With $1,900 × 2 exemptions this gives $90,944.70 taxable, $2,116.61 of tax, and $1,916.61 after the $200 pension-based retirement income credit. The joint filing credit, senior credit and exemption credit don't apply."
-us,scenario_026,child1_chip_eligible,5,llm_error,categorical_eligibility,False,,"NC Health Choice merged into NC Medicaid in 2023. North Carolina now covers school-age children up to 211% FPL, plus a 5% disregard, as Medicaid (Medicaid-expansion CHIP) under the OLDER_CHILD category, not as a separate CHIP program. PolicyEngine finds the 11-year-old Medicaid-eligible at this household income. Because CHIP covers only children who fail Medicaid, that Medicaid eligibility makes the child CHIP-ineligible. Every model that answered treated the 211% band as a separate CHIP program sitting above a lower Medicaid limit. Each one called the child CHIP-eligible instead of routing them to Medicaid first and excluding CHIP."
-us,scenario_026,child1_medicaid_eligible,50,llm_error,thresholds_rates,False,,"NC Health Choice was folded into Medicaid in 2023. Since then, NC's Medicaid older-child category (ages 6–18) has covered children up to 211% FPL, or 216% once the 5% MAGI disregard is added. The household's MAGI is the spouse's $85,209 in wages, which already include overtime, minus the pre-tax traditional 401k and IRA contributions. That is about $82,300, or 2.13× FPL, so Child 1 qualifies. Most models used the federal minimum of 133–138% FPL. Others used 211% but overstated income: they double-counted overtime, used an older, lower FPL, or left out the disregard. A few wrongly treated employer-sponsored insurance as a bar to Medicaid, but only CHIP requires a child to be uninsured."
-us,scenario_026,child2_chip_eligible,5,llm_error,categorical_eligibility,False,,"Child 2 is 11. In the engine, North Carolina's Medicaid OLDER_CHILD category covers this child at the household's income, so the child is Medicaid-eligible. CHIP only covers children who do not qualify for Medicaid, which makes child2_chip_eligible = No. NC folded NC Health Choice into Medicaid, so the income band that the models called ""CHIP, above Medicaid and up to 211-216% FPL"" is Medicaid coverage for a school-age child. All four models that answered found income below the NC children's limit but assigned the child to CHIP instead of Medicaid, and never applied the rule that Medicaid eligibility excludes CHIP."
-us,scenario_026,child2_medicaid_eligible,50,llm_error,thresholds_rates,False,,"MAGI is the spouse's $85,209 in wages minus the $2,778 traditional 401(k) and $130 traditional IRA contributions, which comes to about $82,301. That is 2.13x the 2026 FPL for a household of five. In April 2023 North Carolina folded NC Health Choice into Medicaid, so Medicaid now covers children aged 6-18 up to 211% FPL, or 216% with the 5-point MAGI disregard, and 2.13 falls under that limit. The models answered No for three main reasons: they applied the old 133% FPL limit for ages 6-18 and treated the 211% tier as separate CHIP, they inflated the FPL ratio by using an older (2023-2025) poverty guideline or gross wages, or they dropped the 5-point disregard. A few also wrongly treated employer-sponsored insurance as blocking Medicaid, but having ESI does not bar Medicaid eligibility."
-us,scenario_026,child3_chip_eligible,7,llm_error,categorical_eligibility,False,,"Every model placed 9-year-old Child 3 in a CHIP band that sits above Medicaid, using the old 133%-to-211% FPL NC Health Choice structure. North Carolina moved NC Health Choice into Medicaid in 2023, and PolicyEngine finds Child 3 Medicaid-eligible under the OLDER_CHILD category. A Medicaid-eligible child is excluded from CHIP by construction, so the answer is No however each model computed MAGI. The decisive step is the Medicaid-first ordering with North Carolina's older-child Medicaid limit, not any CHIP income ceiling."
-us,scenario_026,child3_medicaid_eligible,50,llm_error,thresholds_rates,False,,"North Carolina folded its separate CHIP program (NC Health Choice) into Medicaid in April 2023. Since then, children ages 6-18 are covered by Medicaid up to 211% FPL, and the 5-point MAGI disregard raises the effective limit to 216%. Household MAGI is $85,209 in wages minus $2,778 traditional 401(k) and $130 traditional IRA contributions, or about $82,301. That is 2.13x the 2026 FPL for five people, which is under the older-child limit, so child3 is Medicaid-eligible in the OLDER_CHILD category. Seventeen of the 24 models that stated a numerical Medicaid limit used 133% or 138% FPL for this age group, and two said the child qualified for CHIP. Others compared gross wages or an inflated FPL ratio against a 210-211% limit without the 5% disregard, or wrongly treated employer coverage as a bar to Medicaid."
+us,scenario_025,state_income_tax_before_refundable_credits,52,llm_error,thresholds_rates,False,,"Ohio's 2026 schedule taxes nonbusiness income above $26,050 at a base amount of about $332 plus 2.75% of the excess. It is not simply 2.75% of the excess. The correct path takes OAGI ($94,925.30 less a $180.60 medical deduction for $7,300 of premiums and medical costs above 7.5% of FAGI), subtracts two $1,900 exemptions for the over-$80k tier, and taxes $90,944.70 to get $2,116.61. The $200 retirement income credit then gives $1,916.61. Most models dropped the $332 base amount and landed near $1,590 or $1,762. Others used the wrong exemption tier, made up brackets or a standard deduction, skipped the $200 retirement credit, or got the medical deduction wrong (ignored it, added the employer's $21,208 premium, or deducted the $6,500 in full)."
+us,scenario_026,child1_chip_eligible,6,llm_error,categorical_eligibility,False,,"North Carolina moved NC Health Choice (its separate CHIP) into Medicaid in April 2023. Since then, school-age children at this family's income are covered by Medicaid, and PolicyEngine puts child1 in its OLDER_CHILD Medicaid category. CHIP covers only children who don't qualify for Medicaid, so this Medicaid eligibility makes child1 ineligible for CHIP. Every model that answered treated the roughly 211%–216% FPL band as separate CHIP and assumed children's Medicaid stopped at a lower limit, so they all answered Yes."
+us,scenario_026,child1_medicaid_eligible,51,llm_error,thresholds_rates,False,,"Child 1 falls in North Carolina's older-child Medicaid category (ages 6–18). Since NC Health Choice moved into Medicaid in April 2023, the limit for this category is 211% FPL, or 216% FPL after the 5-percentage-point MAGI disregard. Household MAGI is $85,209 in wages minus $2,778 of traditional 401(k) and $130 of traditional IRA contributions, which is $82,301. That equals 2.13 × FPL, so Child 1 qualifies. Most models used the old 133%/138% limit from before 2023. Others overstated income by adding the FLSA overtime premium on top of wages that already include it, by skipping the pre-tax deductions, or by using a 2025 or otherwise wrong FPL. Some wrongly treated employer-sponsored insurance as a bar to Medicaid, when it only bars CHIP."
+us,scenario_026,child2_chip_eligible,6,llm_error,categorical_eligibility,False,,"The engine makes Child 2, who is 11, eligible for Medicaid under the OLDER_CHILD category. PolicyEngine's CHIP eligibility excludes any child who is already Medicaid-eligible. North Carolina folded NC Health Choice into Medicaid, so children aged 6–18 in the income band the models called ""CHIP"" are covered by Medicaid instead of separate CHIP. Each model that answered placed Child 2 in a CHIP band above an assumed lower Medicaid ceiling. None of them first checked whether the older-child Medicaid category already covered Child 2, which makes CHIP unavailable."
+us,scenario_026,child2_medicaid_eligible,51,llm_error,thresholds_rates,False,,"In April 2023, North Carolina folded NC Health Choice (its separate CHIP) into Medicaid. Since then, children ages 6-18 qualify for Medicaid up to about 211% FPL, or about 216% FPL with the 5-point MAGI disregard. The 133%/138% tier no longer sets the cutoff for this age group. Household MAGI is the spouse's $85,209 in wages minus the $2,778 traditional 401(k) and $130 traditional IRA contributions, which gives $82,301. That is 2.13× the 2026 five-person FPL, so it falls under the older-child Medicaid limit, and having employer-sponsored insurance does not block Medicaid eligibility. Most models used the outdated 133%/138% Medicaid tier, treated the 211% tier as CHIP-only or applied it without the disregard, or overstated the FPL ratio by using gross wages or a stale or too-low poverty guideline."
+us,scenario_026,child3_chip_eligible,8,llm_error,categorical_eligibility,False,,"North Carolina folded NC Health Choice, its separate CHIP for children ages 6–18, into Medicaid in 2023. PolicyEngine therefore extends NC's older-child Medicaid income limit through the band the models assigned to CHIP, and it classifies the 9-year-old as Medicaid-eligible under OLDER_CHILD. CHIP covers only children who do not qualify for Medicaid, so the child's Medicaid eligibility alone makes them CHIP-ineligible, whatever the household's FPL percentage. Every model placed the child in a CHIP band above an assumed 133% FPL Medicaid ceiling and never recognized that the Medicaid determination comes first and blocks CHIP."
+us,scenario_026,child3_medicaid_eligible,51,llm_error,thresholds_rates,False,,"North Carolina moved its separate CHIP (NC Health Choice) into Medicaid in April 2023. As a result, the Medicaid older-child category for ages 6–18 now covers children up to 211% FPL, or about 216% with the 5% MAGI disregard. Household MAGI is $85,209 in wages minus $2,778 in traditional 401(k) and $130 in IRA contributions, which gives $82,301, or 2.13× the 2026 FPL for a family of 5. That puts Child 3 inside the Medicaid limit. Most models used the old 133%/138% federal-minimum limit for older children and sent the child to CHIP. The others pushed the household over the limit by using gross wages instead of MAGI, counting overtime twice, using a low or outdated FPL, leaving out the 5% disregard, or treating employer coverage as disqualifying."
us,scenario_026,federal_income_tax_before_refundable_credits,22,llm_error,taxable_income_or_deductions,False,,"2026 law keeps the TCJA structure — personal exemptions repealed, a $32,200 MFJ standard deduction — and adds the deduction for qualified FLSA overtime premiums, so the $7,746.26 premium comes out of taxable income rather than being stacked onto wages that already include it, leaving $42,354.26 taxable and $4,586.51 of tentative tax. The child tax credit is $2,200 per qualifying child in 2026, and its nonrefundable portion offsets the entire liability first, with the $1,700-per-child ACTC as the residual, so $6,600 of credit for three children drives the answer to exactly $0. The wrong answers split into two clusters: models that applied the expired pre-TCJA regime (a reverted ~$16,600 standard deduction, ~$5,200 personal exemptions, a 15% bracket, a $1,000-per-child credit), and models that got the deduction stack roughly right but capped the nonrefundable CTC at $500–$1,100 per child by treating the refundable ACTC as consumed first. Every wrong model ignored the qualified-overtime deduction, and three double-counted the $7,746 premium on top of the $85,209 gross wage figure."
-us,scenario_026,federal_refundable_credits,43,llm_error,taxable_income_or_deductions,False,,"Under OBBBA for 2026, the household's AGI is $82,301: $85,209 in wages minus the $2,778 traditional 401(k) deferral and the $130 traditional IRA deduction. Taxable income is $42,355 after the $32,200 MFJ standard deduction and the $7,746 qualified overtime (FLSA premium) deduction. That gives pre-credit tax of about $4,586.51. The $6,600 CTC ($2,200 × 3) absorbs that tax, and the remaining $2,013.49 is paid as refundable ACTC. It is below the $1,700-per-child cap and the 15%-of-earnings test, and EITC is zero at this AGI. Most models skipped the overtime deduction, assumed the 2017-law $1,000 or $2,000 CTC, or said the tax fully absorbed the credit without checking. A few added deductions or credits that don't apply here, such as auto-loan interest or the Saver's Credit."
+us,scenario_026,federal_refundable_credits,44,llm_error,taxable_income_or_deductions,False,,"For 2026, the OBBBA Child Tax Credit is $2,200 per child, so three children give $6,600. Of that, $1,700 per child ($5,100 total) is the most that can be refundable. AGI is $82,301 ($85,209 wages minus the $2,778 traditional 401(k) and $130 traditional IRA). From that, subtract the $32,200 MFJ standard deduction and the OBBBA qualified overtime deduction for the $7,746 FLSA overtime premium. That leaves about $42,355 of taxable income and $4,586.51 of tax before credits. The unused $2,013.49 of CTC is refundable, well under both the cap and the 15% earned-income formula. EITC and the Saver's Credit are zero because AGI is above both limits. Most models went wrong in one of four ways: they used the pre-OBBBA $2,000 credit or assumed TCJA expired and used $1,000, they skipped the overtime deduction, they deducted the $378 auto-loan interest, or they claimed a Saver's Credit."
us,scenario_026,free_school_meals_eligible,1,llm_error,thresholds_rates,False,,"Free school meals require income within the free-meal limit or categorical eligibility through a qualifying program. This household's countable income of $85,208.82 equals 220% of the $38,680 federal poverty guideline, exceeds even the 185% reduced-price limit, and has no categorical eligibility, so its school-meal tier is paid."
us,scenario_026,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_026,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_026,payroll_tax,11,llm_error,payroll_tax_base,False,,"The only earner, the spouse, has $85,209 in gross wages, and the prompt says that amount already includes overtime. Employee FICA is therefore 6.2% Social Security ($5,282.95) plus 1.45% Medicare ($1,235.53), for $6,518.47. The 2026 Social Security wage cap, the Additional Medicare Tax threshold and any NC employee payroll tax don't apply, since the wages are below both thresholds and NC has no such tax. Most wrong answers added the $7,746 FLSA overtime premium on top of the gross wages, subtracted the head's $21,208 employer insurance premiums from the FICA base, counted NC income tax as payroll tax, or got the arithmetic wrong."
+us,scenario_026,payroll_tax,12,llm_error,payroll_tax_base,False,,"Employee payroll tax here is 7.65% of the one earner's $85,209 gross wages: 6.2% Social Security ($5,282.95), well under the wage base, plus 1.45% Medicare ($1,235.53). There is no Additional Medicare Tax and North Carolina has no employee state payroll tax. The prompt says gross wages already include overtime, so adding the $7,746 FLSA overtime premium counts it twice. Traditional 401(k) deferrals stay in the FICA wage base. The $21,208 employer-sponsored insurance premiums are listed for the head, who has no wages, so they don't reduce the spouse's wage base. Most wrong answers made one of these three base adjustments, added a state tax that doesn't exist, or simply miscalculated."
us,scenario_026,reduced_price_school_meals_eligible,1,llm_error,categorical_eligibility,False,,"Reduced-price school meals require household income at or below 185% of the federal poverty guideline or a qualifying categorical-eligibility pathway. The household's school-meal income is $85,209, equal to 2.20 times the guideline, and it has neither universal-free-meal coverage nor categorical eligibility, so its tier is PAID."
us,scenario_026,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_026,spouse_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -216,7 +216,7 @@ us,scenario_027,local_income_tax,1,parse_contract_failure,missing_output,False,,
us,scenario_027,payroll_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_027,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_027,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_027,snap,50,llm_error,categorical_eligibility,False,,"Connecticut gives broad-based categorical eligibility through TANF non-cash benefits. Gross income of $2,536.25 a month is 1.44 times the poverty guideline, which is under the 200% limit. That waives both the asset test (so the $48,000 doesn't matter) and the 100% FPL net income test. Monthly net income of $2,118 gives a 30% contribution of $636, more than the $546 two-person maximum, so the formula benefit is $0. But a categorically eligible one- or two-person household still gets the minimum allotment: $24 a month, or $288 a year. Most models failed the household on the asset limit or the net income limit, or zeroed the benefit without applying the minimum. A few applied the minimum but used the FY2025 figure of $23."
+us,scenario_027,snap,51,llm_error,categorical_eligibility,False,,"Connecticut's broad-based categorical eligibility (TANF non-cash, 200% FPL gross limit) makes this two-person elderly/disabled household eligible. Its gross income of $2,536.25/month is 1.44× poverty, so the asset test and the 100%-FPL net income test are both waived. The benefit formula gives zero because 30% of the $2,118 net income ($636) exceeds the $546 two-person maximum. Eligible one- and two-person households still get the FY2026 minimum allotment of $24/month, or $288/year. Most models either applied the $4,250–$4,500 elderly/disabled asset limit or the net income test, which BBCE waives, or stopped at a zero formula benefit without applying the minimum allotment. A few used the outdated $23 minimum or left 8% of $298 unrounded."
us,scenario_027,spouse_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_027,spouse_medicaid_eligible,14,llm_error,asset_resource,False,,"For Medicaid, MAGI adds back all non-taxable Social Security. The household's MAGI is therefore $30,160 SSDI + $275 interest = $30,435, which is 1.41x the 2026 two-person FPL. That is above the 138% adult-expansion limit, which PolicyEngine applies without a 5% disregard. The aged/blind/disabled route also fails: the couple's $48,000 in bank assets is far above Connecticut's ABD resource limit and the $3,000 SSI couple limit, so SSI is $0 and the spouse has no SSI-linked or optional-disabled category. medicaid_category is NONE. Models either left Social Security out of MAGI, got the 138% comparison wrong, or assumed blindness or disability alone qualifies without applying the resource and income tests."
us,scenario_027,spouse_medicare_eligible,12,llm_error,age_disability,False,,"Medicare eligibility requires age 65+, ESRD/ALS, or 24 months of the person's own Social Security disability benefit receipt. A disability flag alone does not qualify anyone. The 39-year-old spouse is flagged blind and disabled but has $0 Social Security disability income of their own; the $30,160 SSDI belongs to the head. The spouse therefore has no Medicare pathway. The models mistook the disability flag for SSDI entitlement, and some credited the head's SSDI to the spouse."
@@ -225,33 +225,33 @@ us,scenario_027,ssi,1,parse_contract_failure,missing_output,False,,All wrong res
us,scenario_027,state_income_tax_before_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_027,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_027,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_028,child1_chip_eligible,21,llm_error,health_coverage,False,,"Every model that answered treated CHIP as a simple income band. Each put the family somewhere above Pennsylvania's child Medicaid limit and below its roughly 314% FPL CHIP limit, then called the 10-year-old eligible. In PolicyEngine, Medicaid is denied (category NONE), and the child also fails CHIP's own child test. Age 10 passes, so the failure is in the eligibility comparison, not age. The fact the models overlooked is that every child is already covered by employer-sponsored insurance. Under 42 U.S.C. 1397jj(b)(1)(C) and 42 CFR 457.310(b)(2), a child covered by a group health plan is not a targeted low-income child, so CHIP eligibility is 0. Several models went further and explicitly called ESI an enrollment fact that does not affect eligibility."
+us,scenario_028,child1_chip_eligible,22,llm_error,health_coverage,False,,"Child 1 is age 10 and has no Medicaid pathway (engine category NONE), so CHIP eligibility depends entirely on PolicyEngine's CHIP child test, and Child 1 fails that test. Every model that answered decided eligibility on age plus a comparison of income with Pennsylvania's roughly 314% FPL ceiling. None of them applied the remaining CHIP child requirement: the child is listed as having employer-sponsored insurance, and CHIP covers only targeted low-income children who are not covered by a group health plan. Three models explicitly assumed that existing employer coverage does not affect eligibility. Since the CHIP child test fails and there is no Medicaid route, is_chip_eligible is False."
us,scenario_028,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,child1_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,child1_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"The decisive step is assigning the child to a Pennsylvania Medicaid eligibility category rather than treating the broader Medicaid/CHIP coverage continuum as Medicaid. At 1.82 times FPL, the child satisfies none of the Medicaid pathways evaluated by the engine, has no disability pathway or SSI receipt, and therefore has medicaid_category NONE and is not Medicaid eligible."
us,scenario_028,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,child1_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC categorical eligibility for children ends at the fifth birthday; it does not extend to school-age children. Child 1 is age 10, so the child fails the categorical age requirement regardless of household income."
-us,scenario_028,child2_chip_eligible,22,llm_error,health_coverage,False,,"The models treated being under 19 with income around 185–190% FPL, above PA child Medicaid and below PA's roughly 314–319% CHIP ceiling, as all that CHIP eligibility requires. PolicyEngine's CHIP child test returns False for child2. At age 10 the age requirement is met, so the failure comes from CHIP's own child criteria, not from Medicaid. Child2 is also listed with employer-sponsored insurance. A targeted low-income child under 42 USC 1397jj(b)(1)(C) must not be covered by a group health plan, and most models either ignored this input or said it does not count against eligibility. The engine's is_chip_eligible therefore resolves to False even though Medicaid eligibility is also False."
+us,scenario_028,child2_chip_eligible,23,llm_error,health_coverage,False,,"Child 2 is 10, so CHIP's age test (under 19) passes. Household MAGI of about $60,010 for a family of four is roughly 185% FPL, which means the child is not Medicaid-eligible (engine category NONE). The CHIP child test still fails, and the household fact that decides it is that every child already has employer-sponsored insurance. CHIP only covers targeted low-income children who are not covered by a group health plan (42 U.S.C. 1397jj(b)(1)(C)). Every model that answered stopped once income fell inside PA's CHIP band (up to about 314% FPL), and several said outright that existing ESI does not affect eligibility."
us,scenario_028,child2_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,child2_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,child2_medicaid_eligible,6,llm_error,thresholds_rates,False,,"A 10-year-old Pennsylvania dependent at 1.82 times FPL does not satisfy any Medicaid eligibility pathway evaluated by PolicyEngine, so the Medicaid category is NONE. CHIP’s higher income ceiling is a separate program and cannot establish Medicaid eligibility."
us,scenario_028,child2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,child2_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC child eligibility is limited to children under age five and also requires household income at or below 185% of the federal poverty guideline. Child 2 is age 10, and the household's approximately $61,277 of income exceeds the applicable four-person limit, so both the categorical age test and the income test fail."
-us,scenario_028,child3_chip_eligible,21,llm_error,health_coverage,False,,"Child 3 meets CHIP's age and income tests. The child is 10, and household MAGI is $60,010, which is 182% of the $33,000 guideline: above PA's 138% Medicaid limit for children and below the 319% CHIP limit. The deciding rule is the coverage test. The prompt says Child 3 has employer-sponsored insurance. A child covered by a group health plan is not a targeted low-income child under 42 U.S.C. 1397jj(b)(1)(C) and 42 CFR 457.310(b)(2)(ii), so PolicyEngine's disqualifying-coverage check sets CHIP eligibility to 0. Every model that answered stopped after the age and income checks, and several said outright that ESI does not matter. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (health_coverage). Child 3 has employer-sponsored group coverage, which bars CHIP; the household's income passes Pennsylvania's CHIP limit, so the judge's income-limit hypothesis fails. Reference affirmed (42 U.S.C. 1397jj(b)(1)(C); 42 CFR 457.310(b)(2)(ii); Pennsylvania CHIP state plan 4.1.7)."
+us,scenario_028,child3_chip_eligible,22,llm_error,health_coverage,False,,"Child 3 passes CHIP's age test (age 10). They also pass the income test: MAGI is $60,010, which is 182% of the $33,000 FPL. That is above Pennsylvania's 138% Medicaid limit for children and below the 319% CHIP limit (314% plus the 5-point disregard). The deciding fact is that the prompt lists Child 3 as having employer-sponsored insurance. A child covered by a group health plan is not a ""targeted low-income child"" (42 U.S.C. 1397jj(b)(1)(C); 42 CFR 457.310(b)(2)(ii)), so PolicyEngine's disqualifying-coverage test fails and CHIP eligibility is 0. Every model that answered stopped after the age and income tests. Several also stated outright that ESI does not bar CHIP eligibility. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (health_coverage). Child 3 has employer-sponsored group coverage, which bars CHIP; the household's income passes Pennsylvania's CHIP limit, so the judge's income-limit hypothesis fails. Reference affirmed (42 U.S.C. 1397jj(b)(1)(C); 42 CFR 457.310(b)(2)(ii); Pennsylvania CHIP state plan 4.1.7)."
us,scenario_028,child3_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,child3_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,child3_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Pennsylvania Medicaid and CHIP are separate eligibility outputs, so CHIP coverage at higher income levels does not establish Medicaid eligibility. At 1.82 times FPL, child3 exceeds every applicable Medicaid income threshold and qualifies under no Medicaid category; employer-sponsored insurance and zero SSI provide no alternative Medicaid pathway."
us,scenario_028,child3_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,child3_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC child eligibility is limited to children under age five, so a 10-year-old fails the categorical age requirement. The household also has approximately $61,277 of income, exceeding the applicable 185%-of-poverty WIC income limit; Child 3 therefore fails both the categorical and income tests."
us,scenario_028,federal_income_tax_before_refundable_credits,20,llm_error,credit_phaseout,False,,"Under 2026 law (OBBBA made the TCJA structure permanent), the HoH filer has AGI of $60,010. Child support is not taxable. The $24,150 standard deduction leaves $35,860 of taxable income and $3,949.20 of tax. The full $2,200-per-child CTC ($6,600) is applied first as a nonrefundable credit up to that liability, which zeroes it out. Only the leftover $2,650.80 is refundable ACTC. The wrong answers come from three mistakes: assuming TCJA expired (personal exemptions and a $1,000 CTC), carving the refundable ACTC out of the CTC before applying the nonrefundable portion, or applying only part of the CTC against the liability."
-us,scenario_028,federal_refundable_credits,46,llm_error,credit_phaseout,False,,"Under 2026 OBBBA law, the CTC is $2,200 per child, so $6,600 total. The household's AGI is $60,010; subtracting the $24,150 head-of-household standard deduction leaves taxable income of $35,860. The 2026 brackets put pre-credit tax at $3,949.20, which leaves $2,650.80 of unused CTC. That amount is refundable because it is below both the $5,100 cap ($1,700 per child) and 15% of earnings over $2,500 ($8,625). The 3-child EITC has not fully phased out at this income: $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. Most models got this wrong in one of four ways: they used the old $2,000 (or $1,000) per-child CTC, they ignored the unused-CTC limit and claimed the full $5,100 cap, they wrongly said the EITC was fully phased out below $60,010, or they used the wrong EITC maximum or phase-out start."
-us,scenario_028,free_school_meals_eligible,51,llm_error,state_local_rule,False,,"The models that returned No applied only the federal NSLP/SBP test. Income is about 186% of the poverty guideline, above the 130% free limit, and there is no SNAP or TANF categorical eligibility, so the models concluded no free meals. They all missed Pennsylvania's universal free school breakfast (Act 33 of 2023, 24 P.S. § 13-1337.1(c.2)). It gives free breakfast to students at participating schools regardless of income. PolicyEngine assumes the three 10-year-olds attend such schools and values the free breakfast at $1,112.40 a year ($2.06 per child per day over 180 days), so free school meal support is positive and the output is 1, even though the income-based tier for lunch is PAID."
+us,scenario_028,federal_refundable_credits,47,llm_error,credit_phaseout,False,,"Under the 2026 rules (OBBBA), the CTC is $2,200 per child, so three children give $6,600. HoH taxable income is $60,010 − $24,150 = $35,860, and pre-credit tax is $1,770 + 12% × $18,160 = $3,949.20. The refundable CTC is the unused $2,650.80, which is below both the $5,100 per-child cap (3 × $1,700) and the $8,625 earned-income limit. The 3-child EITC has not fully phased out at $60,010 (it ends near $62,973) and equals $8,231 − 21.06% × ($60,010 − $23,890) = $624.13. Models went wrong in four main ways: they used the old $2,000 (or $1,000) per-child CTC, took the $5,100 cap without limiting it to the unused CTC, assumed the CTC was fully absorbed by tax, or set the EITC to zero or used the wrong phaseout start."
+us,scenario_028,free_school_meals_eligible,52,llm_error,state_local_rule,False,,"The result depends on a Pennsylvania rule, not the federal income test. Under Act 33 of 2023 (24 P.S. § 13-1337.1(c.2)), Pennsylvania gives every student at participating schools a free breakfast regardless of household income. PolicyEngine therefore puts breakfast in the free tier for the three school-age children, worth $2.06 per child per day over 180 days, or $1,112.40. That makes free school meal support positive even though income (1.82 times the poverty guideline, the reduced-price tier) disqualifies the household from federally free lunch. Every model that answered applied only the federal 130% FPL free-meal test or the SNAP/TANF test and never applied Pennsylvania's universal free breakfast."
us,scenario_028,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_028,payroll_tax,39,llm_error,state_local_rule,False,,"The reference is employee Social Security ($3,720) plus Medicare ($870) plus Pennsylvania's mandatory employee unemployment compensation (UC) withholding of 0.07% on all wages, which is uncapped ($60,000 × 0.0007 = $42), for $4,632. Almost every model stopped at FICA ($4,590) and said Pennsylvania has no employee-side payroll tax. Pennsylvania is one of the few states that withholds UC tax from employees. One model correctly reached $4,632 in its reasoning but submitted a different value. Another applied the employer's $10,000 UC taxable wage cap to the employee withholding."
+us,scenario_028,payroll_tax,40,llm_error,state_local_rule,False,,"Pennsylvania is one of the few states that requires an employee unemployment compensation contribution. It is 0.07% of all gross wages with no wage cap, which comes to $42 on $60,000. The reference adds this to $3,720 of employee Social Security tax and $870 of employee Medicare tax, for $4,632. Almost every model stopped at federal FICA ($4,590), and many said outright that Pennsylvania has no employee state payroll tax. The other misses were a wrongly capped Pennsylvania contribution, an arithmetic error, a reported value that contradicted the model's own working, and two missing answers."
us,scenario_028,reduced_price_school_meals_eligible,9,llm_error,thresholds_rates,False,,"For school meals, countable income includes the $1,267 of child support along with $60,000 of wages and $10 of interest, for a total of $61,277. The 185% reduced-price limit for a household of four, using the $33,000 2026 poverty guideline, is $61,050, so the household is $227 over the limit. Wages plus interest alone ($60,010, 1.82 times the guideline) would fall in the reduced-price band, and that is the trap. The models either left out the child support or set the 185% limit above $61,277 by using an inflated guideline."
us,scenario_028,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_028,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -273,12 +273,12 @@ us,scenario_030,head_wic_eligible,2,llm_error,household_unit_or_filing_status,Fa
us,scenario_030,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_030,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax equals 6.2% Social Security plus 1.45% Medicare on the $13,000 wage base. Those components are $806.00 and $188.50, totaling $994.50 without whole-dollar rounding."
us,scenario_030,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_030,snap,51,llm_error,categorical_eligibility,False,,"SNAP counts the $12,000 of financial assistance as unearned income and excludes the educational assistance. That puts gross income at $2,083.33/month, or 1.60× the poverty guideline. This is above the standard 130% gross test but below Texas's 165% FPL broad-based categorical eligibility (TANF non-cash) limit. Categorical eligibility waives both the gross and net income tests. Expected contribution is $498 after rounding 30% of $1,658 up to the next dollar, which is more than the $298 one-person maximum, but an eligible 1–2 person household still gets the minimum allotment of 8% × $298 ≈ $24/month, or $288/year. The models that answered $0 either applied the 130% gross test or the net income test without Texas BBCE, or stopped at ""contribution exceeds the maximum"" and skipped the minimum allotment. The models with positive amounts either left out the financial assistance or counted the unborn child as a second household member. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (categorical_eligibility). The frozen 287.68 is regenerated as 288.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze): The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published). Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C))."
+us,scenario_030,snap,52,llm_error,categorical_eligibility,False,,"The SNAP unit has one person; the pregnancy does not add a member. Countable income is $13,000 in wages plus $12,000 in financial assistance, which is $2,083.33 a month. Educational assistance is excluded. Texas broad-based categorical eligibility through TANF non-cash benefits uses a 165% FPL gross limit and waives the net income test. Gross income is 1.6× FPL under the $15,650 guideline, so the household is eligible. Net income is $1,658, so the expected contribution is $498, which is more than the $298 maximum allotment. An eligible one-person household still gets the $24 minimum allotment, so the annual benefit is $288. Most models either (a) applied the 130% gross or 100% net test, or used a stale FPL, and declared the household ineligible, (b) correctly found 30% of net income above the maximum but skipped the minimum allotment, or (c) left out the $12,000 in financial assistance and computed a larger benefit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (categorical_eligibility). The frozen 287.68 is regenerated as 288.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9162 (merged 2026-07-28), after the reference freeze): The minimum benefit is 8% of the maximum allotment for a household of one, rounded to the nearest whole dollar ($24 a month for FY2026, as USDA published). Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 U.S.C. 2017(a); 7 CFR 273.10(e)(2)(ii)(C))."
us,scenario_030,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_030,tanf,2,llm_error,categorical_eligibility,False,,Texas TANF requires a family with a dependent child; pregnancy alone does not turn the unborn child into a qualifying dependent child or create a two-person assistance unit for this annual TANF calculation. The single adult therefore has no qualifying child and receives zero TANF.
us,scenario_031,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"The $1,165 alimony expense reduces $14,408 of IRA and pension income to $13,243 of adjusted gross income, while Social Security remains nontaxable. The applicable single-filer standard deduction, including the age-65 addition, exceeds that income, producing zero taxable income and therefore zero federal income tax before refundable credits."
-us,scenario_031,head_medicaid_eligible,48,llm_error,state_local_rule,False,,"At 67 the head is aged, and that alone puts them in California's optional senior-or-disabled Medi-Cal category. No SSI receipt or disability is needed, and Medicare eligibility bars only the 19–64 expansion group. Income is counted under SSI rules: Social Security in full plus pension and IRA comes to $23,853.47, and alimony paid is not subtracted. California's $230/month ($2,760/year) disregard replaces SSI's $20 general exclusion, which brings countable income down to $21,093.47. That is under the limit of 138% of the 2026 FPL, $22,024.80. Assets of $4,200 are far below California's $130,000 limit. Nearly every model compared gross income ($23,853) to a 138% or 100% FPL limit, or used only the $20 exclusion, and so missed the $230 California disregard. Others wrongly treated Medicare age, a $2,000 asset limit, or the lack of a disability as disqualifying."
-us,scenario_031,snap,48,llm_error,categorical_eligibility,False,,"The household's gross income is about $1,988 a month, which is under California's 200% FPL limit for Broad-Based Categorical Eligibility (about $2,608 a month). That makes it categorically eligible for CalFresh, so both the net income test and the asset test are waived. The regular benefit formula gives $0, because 30% of net income ($484) is more than the $298 maximum allotment. Even so, a categorically eligible one-person household still gets the FY2026 minimum allotment of $24 a month (8% of $298, rounded), which comes to $288 for the year. Most models either applied the net or gross income test without the BBCE waiver, or stopped at the $0 formula result and never applied the minimum benefit. Two models used the right pathway but plugged in the old $23 minimum."
+us,scenario_031,head_medicaid_eligible,49,llm_error,state_local_rule,False,,"At 67 the head qualifies categorically for California's non-MAGI aged-and-disabled pathway without SSI receipt or a disability. That pathway counts income under SSI rules, so Social Security, the pension and the IRA distribution are counted in full ($23,853.47) and alimony paid is not subtracted. California then applies a $230 monthly disregard ($2,760 a year) in place of SSI's $20 general exclusion, which leaves $21,093.47. That is under the 2026 limit of 138% FPL for one person ($22,024.80), and the $4,200 in assets is far below California's $130,000 individual resource limit. Every wrong model either compared gross income or income less only the $20 exclusion to the limit, or used the wrong limit or asset test, or wrongly held that Medicare eligibility or the lack of SSI receipt or a disability rules out Medicaid."
+us,scenario_031,snap,49,llm_error,categorical_eligibility,False,,"California's broad-based (Modified) Categorical Eligibility uses a gross income limit of 200% of the poverty guideline. The household's $1,988 a month is under that limit ($2,608), so it is categorically eligible. That waives the 100% FPL net income test and the federal asset test. The formula still gives less than zero ($298 maximum minus 30% of $1,611 net income, which is $484). But an eligible one- or two-person household always gets the minimum allotment under 7 U.S.C. 2017(a): 8% of the FY2026 one-person maximum, rounded to $24 a month, or $288 a year. Most models applied the net income test, or zeroed out the negative benefit, without considering categorical eligibility or the minimum. A few used the outdated $23 minimum or left 8% of $298 unrounded."
us,scenario_031,state_income_tax_before_refundable_credits,1,llm_error,credit_phaseout,False,,"California first computes $75.37 of tax on $7,537 of taxable income, then applies $312.93 of nonrefundable credits, including the aged-or-blind exemption credit. Those credits fully eliminate the liability, so state income tax after nonrefundable credits and before refundable credits is $0."
us,scenario_032,child1_chip_eligible,13,llm_error,health_coverage,False,,"CHIP is residual by statute: 42 U.S.C. 1397jj(b)(1)(A) and 42 CFR 457.310 exclude from ""targeted low-income child"" any child who is eligible for Medicaid, so the CHIP question is settled by first running the Medicaid test. Minnesota's Medical Assistance covers children ages 6-18 — PolicyEngine's OLDER_CHILD category — far above this household's $42,664 for three people (~160% FPL), so the child is Medicaid-eligible and thereby barred from CHIP. Every wrong model instead applied a standalone CHIP income ceiling, most of them quoting Minnesota's 275%/283% FPL child limits, which are the Medical Assistance and MinnesotaCare bands rather than CHIP bands. They read ""income below the child-coverage limit"" as qualifying when it is precisely what disqualifies: several stated the Medicaid fact outright and then used it backwards."
us,scenario_032,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -288,13 +288,13 @@ us,scenario_032,child1_medicare_eligible,2,llm_error,categorical_eligibility,Fal
us,scenario_032,child1_wic_eligible,6,llm_error,categorical_eligibility,False,,"WIC child eligibility ends at the child’s fifth birthday; a six-year-old is outside the federally eligible categorical group. Income eligibility cannot override this age restriction, so the child is ineligible regardless of whether household income falls below 185% of the federal poverty guidelines."
us,scenario_032,federal_income_tax_before_refundable_credits,3,llm_error,thresholds_rates,False,,"The decisive rule is the 2026 federal parameter regime: the household receives a $2,200 CTC, split into $500 nonrefundable and $1,700 refundable. The $500 nonrefundable portion fully eliminates pre-credit income tax, while the refundable CTC and $4,427 EITC are excluded from federal_income_tax_before_refundable_credits."
us,scenario_032,federal_refundable_credits,40,llm_error,thresholds_rates,False,,"The household files jointly with one qualifying child. Earned income is $29,000. AGI is about $30,916 because only $1,916 of the Social Security benefits is taxable, using the $32,000 MFJ base amount. That AGI is below the 2026 MFJ one-child EITC phase-out start, so the EITC is the full $4,427 maximum. The 2026 MFJ standard deduction of $32,200 wipes out all tax liability, so the CTC is refundable up to the $1,700 per-child cap under the post-OBBBA law, which makes the $2,200 CTC permanent. Most models missed by assuming the TCJA sunset ($1,000 ACTC), using the wrong refundable cap ($1,800, $2,200 or uncapped), applying the single-filer EITC phase-out start without the joint-filer increase, or misstating the 2026 one-child EITC maximum."
-us,scenario_032,free_school_meals_eligible,27,llm_error,state_local_rule,False,,"Minnesota has had universal free school meals since the 2023-24 school year (Minn. Stat. §124D.111, as amended in 2023). PolicyEngine encodes this as state_has_universal_free_school_meals = True, so every K-12 child in the state gets the FREE tier regardless of income. The household is at 1.56 times the FPG, above the federal 130% free-meal limit, and has no SNAP/TANF categorical eligibility, but neither fact matters because the state rule overrides the income test. Every model that answered applied only the federal NSLP 130% FPL test and never checked Minnesota's universal free-meals law."
+us,scenario_032,free_school_meals_eligible,28,llm_error,state_local_rule,False,,"The household's income is 1.56 times the federal poverty guideline, which is above the federal 130% free-meal cutoff, and the household has no SNAP/TANF categorical eligibility. Minnesota runs a universal free school meals program (Minn. Stat. 124D.111, in effect since 2023), so PolicyEngine sets the school meal tier to FREE for every K-12 student in the state regardless of income. The 6-year-old is in K-12 school, so the household gets positive free-meal support. Every model that answered applied only the federal NSLP income test and missed Minnesota's universal program."
us,scenario_032,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_032,head_medicaid_eligible,16,llm_error,taxable_income_or_deductions,False,,"Medicaid MAGI is AGI plus the non-taxable portion of Social Security benefits (42 CFR 435.603(e)). The head's full $13,664 of Social Security dependent benefits therefore counts along with the $29,000 in wages, for a MAGI of $42,664. That is about 156% of the 3-person FPL ($27,320), which is above Minnesota's 138% FPL Medical Assistance limit. Minnesota's limit for parents and caretakers is the same 133%+5% standard as for expansion adults, and the head is not aged, disabled, pregnant, or receiving SSI, so no category applies. Most models counted only wages, or wages plus the taxable portion of Social Security (about $30,916, roughly 113% FPL). Two models counted the full $42,664 but then invented a higher limit for parents, and one inflated the household size."
us,scenario_032,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_032,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is person-specific and requires the applicant to be pregnant, postpartum, breastfeeding, an infant, or a child under age five. A 50-year-old head with all relevant unlisted statuses set to false fails categorical eligibility regardless of household income or another household member's circumstances."
-us,scenario_032,payroll_tax,44,prompt_ambiguity,state_local_rule,False,,"The reference includes $1,798 in Social Security tax, $420.50 in Medicare tax and $127.60 for the Minnesota Paid Leave employee contribution, which starts January 1, 2026. That contribution is 0.44% of $29,000, the employee half of the 0.88% premium. PolicyEngine assumes the employer withholds the full employee share. Minnesota also lets the employer pay that share itself, and the prompt's phrase ""mandatory employee state payroll taxes"" doesn't say which case applies. One reading counts the withheld Paid Leave share, giving $2,346.10. The other leaves it out because the employer can choose not to withhold it, giving FICA only at $2,218.50. Most models took the second reading, and many of them also said wrongly that Minnesota has no employee payroll levy in 2026. The outlier answers come from wrong rates, inconsistent numbers or invented adjustments. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 2,346.10; the alternative is 2,218.50. The prompt counts mandatory employee state payroll taxes; Minnesota Paid Leave contributions are the employer's, which may deduct the employee share from wages but need not (Minn. Stat. 268B.14, subds. 1, 3 and 5a), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($127.60); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
-us,scenario_032,reduced_price_school_meals_eligible,29,llm_error,state_local_rule,False,,"Since the 2023-24 school year, Minnesota has provided universal free school meals to all students regardless of income (Minn. Stat. §124D.111). PolicyEngine therefore puts this household in the FREE school_meal_tier, and reduced-price eligibility is 0 because the household gets the better free tier. Every model that answered applied only the federal NSLP income bands: income at about 156% FPL, between 130% and 185%, means reduced-price. None of them checked for Minnesota's state universal free-meals override. The household's income being inside the reduced-price band does not matter once the state universal program assigns the FREE tier."
+us,scenario_032,payroll_tax,45,prompt_ambiguity,state_local_rule,False,,"Every model got federal FICA right: 6.2% Social Security ($1,798) plus 1.45% Medicare ($420.50), for $2,218.50. The $127.60 difference comes from Minnesota Paid Leave, which began collecting premiums on January 1, 2026. PolicyEngine adds the 0.44% employee share on the head's $29,000 of wages and assumes the employer withholds the maximum employee share. The question asks only for ""mandatory employee state payroll taxes,"" and Minnesota lets the employer pay that share instead. So there are two valid readings: (1) include the 0.44% contribution, assuming maximum employee withholding, or (2) leave it out because the employer can choose to pay it. Models that answered about $2,218.50 took the second reading. Only the deepseek-v4-pro models included the contribution, and they used an outdated 0.35% rate. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 2,346.10; the alternative is 2,218.50. The prompt counts mandatory employee state payroll taxes; Minnesota Paid Leave contributions are the employer's, which may deduct the employee share from wages but need not (Minn. Stat. 268B.14, subds. 1, 3 and 5a), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($127.60); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
+us,scenario_032,reduced_price_school_meals_eligible,30,llm_error,state_local_rule,False,,"Minnesota has run universal free school meals since the 2023-24 school year. PolicyEngine therefore puts every Minnesota student in the FREE school_meal_tier, whatever the household's income. Free and reduced-price tiers are mutually exclusive, so reduced_price_school_meals_eligible is 0 even though the household's 1.56 FPG ratio falls in the federal 130–185% reduced-price band. Every answering model used only the federal NSLP income bands and never applied Minnesota's universal free meals override. The household is also not categorically eligible through SNAP or TANF."
us,scenario_032,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_032,snap,8,llm_error,categorical_eligibility,False,,"SNAP eligibility ends at the gross-income screen for this three-person Minnesota household: $29,000 of wages plus $13,664 of Social Security dependent benefits produces $42,664 of annual gross income, above the applicable limit. Take-up assumptions and benefit-allotment calculations do not create eligibility after that screen fails, so the annual benefit is $0."
us,scenario_032,spouse_chip_eligible,4,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the person to be ineligible for Medicaid, not merely under age 19 and below the CHIP income ceiling. The 18-year-old spouse qualifies for Minnesota Medicaid under the OLDER_CHILD category, so that Medicaid eligibility precludes CHIP eligibility without regard to whether household income is below the CHIP threshold."
@@ -302,8 +302,8 @@ us,scenario_032,spouse_medicaid_eligible,18,llm_error,categorical_eligibility,Fa
us,scenario_032,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_032,spouse_wic_eligible,3,llm_error,categorical_eligibility,False,,"WIC does not cover every low-income woman of childbearing age or every adult living with a child. An adult woman must be pregnant, breastfeeding, or within the applicable postpartum period; the spouse has none of these statuses because all unlisted statuses are false, so she fails categorical eligibility regardless of the income test."
us,scenario_032,state_income_tax_before_refundable_credits,7,llm_error,state_local_rule,False,,"Minnesota starts from federal AGI ($29,000 wages plus the $1,916 of Social Security made taxable by the §86 provisional-income formula = $30,916), then applies two Minnesota-specific subtractions the models ignored or undersized: the Minnesota Social Security benefit subtraction, which removes the entire $1,916 for a joint filer at this income (far below the phase-out range), and the Minnesota dependent exemption for Child 1 (roughly $5,300, indexed). That leaves $29,000 of Minnesota income against a 2026 joint standard deduction of roughly $30,700 plus the dependent exemption, so Minnesota taxable income is zero and tax before refundable credits is $0. Every wrong answer pairs a too-small joint standard deduction with a missing Social Security subtraction and a missing dependent exemption, manufacturing a few hundred to a few thousand dollars of phantom taxable income taxed at the 5.35% first-bracket rate."
-us,scenario_032,state_refundable_credits,51,llm_error,thresholds_rates,False,,"Since 2023, Minnesota's refundable credit has been one combined Child and Working Family Credit. Its 2026 child component is $1,800 per qualifying child. For this household, the working-family component is 4% of earnings up to $9,690, or $387.60, with no additional amount for its child under 18. The combined credit phases out at 12% of the larger of earnings and AGI above the $38,770 joint-filer threshold. AGI here is $30,916 and earnings are $29,000, so the full $2,187.60 is paid. Common mistakes included calculating the Working Family Credit as a percentage of federal EITC; dropping one of the two components; using 2023 amounts and stopping at $2,100; and saying Minnesota has no refundable credit at all."
-us,scenario_033,federal_income_tax_before_refundable_credits,40,prompt_ambiguity,taxable_income_or_deductions,False,,"The reference uses 2026 OBBBA law. AGI is $83,451.23: $79,000 wages, $775 interest, the $243 state refund, 85% of Social Security ($16,320), minus the $11,400 SSTB loss and the $1,487 HSA deduction. From that it subtracts $47,500: the $32,200 MFJ standard deduction, 2 × $1,650 additional aged amounts, and the 2 × $6,000 senior deduction. That leaves $35,951.23 of taxable income, taxed as $2,480 (10% up to $24,800) plus 12% on the rest, for $3,818.15. Models went wrong in a few common ways. Some assumed the TCJA sunset (personal exemptions and a 15% bracket). Some left out the $12,000 senior deduction or used 2024/2025 deduction and bracket amounts. Some dropped the $243 refund or the $11,400 loss. Others took deductions the engine doesn't apply, such as $1,225 of auto-loan interest or an overtime deduction inferred from the hourly wage. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,818.15; the alternative is 3,788.99. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
+us,scenario_032,state_refundable_credits,52,llm_error,thresholds_rates,False,,"Minnesota's 2023 reform replaced the old EITC-percentage Working Family Credit with a single refundable Child and Working Family Credit. It has two parts: a per-child credit, which was $1,750, and a working-family part worth 4% of earnings with a flat cap, which was $350. Both are indexed. The reference total, $2,187.60, equals $2,100 × about 1.0417. That works out to an indexed child credit of about $1,823 plus the capped working-family part of about $365 (4% of $29,000 is $1,160, well above the cap). There is no phaseout, because AGI of $30,916 (wages plus $1,916 of taxable Social Security) is below the indexed joint threshold of about $36,460. Most wrong answers came from one of three mistakes: keeping the unindexed $1,750/$350 figures, adding the repealed EITC-based WFC on top of the child credit, or missing the CWFC entirely."
+us,scenario_033,federal_income_tax_before_refundable_credits,41,prompt_ambiguity,taxable_income_or_deductions,False,,"Two-thirds of models got AGI right at $83,451: wages $79,000, interest $775, the $243 state refund, 85% of Social Security ($16,320), minus the $11,400 SSTB loss and the $1,487 HSA deduction. The answers split on the deductions. The reference subtracts the 2026 OBBBA standard deduction of $35,500 ($32,200 plus 2×$1,650 for age) and the $6,000-per-senior deduction ($12,000 total). That gives taxable income of $35,951.23, and tax is $2,480 + 12% × $11,151.23 = $3,818.15. Most wrong answers made one of these errors: assumed TCJA had sunset (personal exemptions and a 15% bracket), left out the $12,000 senior deduction, used outdated standard deduction or bracket figures, or took deductions the facts don't support (auto-loan interest with no qualifying new vehicle, an overtime deduction inferred from the hourly wage, or dropping the $243 refund). Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,818.15; the alternative is 3,788.99. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
us,scenario_033,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_033,payroll_tax,11,llm_error,other,False,,"This is a simple calculation: employee FICA of 7.65% (6.2% Social Security plus 1.45% Medicare) on combined wages of $79,000 gives $6,043.50. Both spouses earn well under the Social Security wage base, the $250,000 joint Additional Medicare Tax threshold doesn't apply, South Dakota has no employee state payroll tax, and the negative self-employment income doesn't touch employee payroll tax. Almost every model either reached $6,043.50 and then submitted a different number (arbitrary rounding, an invented adjustment, or an unexplained replacement) or multiplied 7.65% × $79,000 wrong. One model gave no answer."
us,scenario_033,self_employment_tax,1,llm_error,payroll_tax_base,False,,"Self-employment tax is a nonnegative liability assessed only on positive net earnings from self-employment. The listed negative SSTB amount does not create a negative self-employment tax or refundable tax offset, so the self-employment tax base and liability are both zero."
@@ -316,7 +316,7 @@ us,scenario_036,state_refundable_credits,1,parse_contract_failure,missing_output
us,scenario_037,federal_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"Under OBBBA in 2026, the $9,629 FLSA overtime premium qualifies for the below-the-line qualified overtime deduction. The single standard deduction is $16,100, and the 10% bracket ends at $12,400. Starting from AGI of $41,539 (wages minus the $386 401(k) and $18 IRA), taxable income is $15,810, and tax is $1,240 + 12% × $3,410.43 ≈ $1,649.25. Most models missed the overtime deduction, which is worth about $1,155 of tax, or assumed TCJA had sunset. Others subtracted the ESI premium figure from the stated gross wages. Several used AGI reduced by the overtime deduction and claimed a Saver's Credit, but the true AGI of $41,539 is above the 2026 single Saver's Credit limit."
us,scenario_037,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"A 44-year-old single filer with no dependent children or qualifying education expenses and $41,943 of wages receives neither EITC nor refundable CTC. All other refundable federal credit components evaluated to zero, so federal refundable credits total $0."
us,scenario_037,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"North Carolina’s Medicaid expansion does not make every adult aged 19–64 eligible; the expansion pathway applies an income limit. At 2.60 times FPL, the head exceeds the applicable MAGI threshold and qualifies through no other Medicaid category, so the eligibility category is NONE."
-us,scenario_037,payroll_tax,15,llm_error,payroll_tax_base,False,,"The FICA base is the full $41,943 of gross wages, so 6.2% Social Security ($2,600.49) plus 1.45% Medicare ($608.18) gives $3,208.67. The household facts never say the $5,789 of employer-sponsored insurance premiums is paid through a Section 125 salary-reduction plan, and the prompt says unlisted status inputs are false. Even so, the largest group of models (eight) subtracted that amount from FICA wages and landed on $2,765.78. The other wrong answers started from the right 7.65% × gross-wages calculation and then broke it by changing the number arbitrarily, rounding loosely, giving a placeholder, or giving no answer at all."
+us,scenario_037,payroll_tax,16,llm_error,payroll_tax_base,False,,"The reference applies the 7.65% employee FICA rate to the full $41,943 of gross wages: 6.2% Social Security ($2,600.49) plus 1.45% Medicare ($608.18), for $3,208.67. The household facts never say the $5,789 employer-sponsored insurance premium is a pre-tax Section 125 salary reduction, and the prompt says unlisted statuses default to false. Traditional 401(k) deferrals stay in FICA wages under IRC §3121(v)(1)(A). Most models wrongly subtracted the ESI premium, which lowers the base to $36,154 and the tax to $2,765.78. The rest subtracted the 401(k) deferral, made arithmetic slips, applied unexplained adjustments, or gave no real answer."
us,scenario_037,state_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"North Carolina starts from federal AGI. Here that is the stated gross wages of $41,943 minus the traditional 401(k) contribution ($386) and the deductible traditional IRA contribution ($18), which gives $41,539 using the displayed whole-dollar amounts. The published derivation reports AGI of $41,539.45. Subtracting the $12,750 single standard deduction leaves $28,789.45, and the flat 3.99% rate gives $1,148.70. The most common mistake was also subtracting the $5,789 employer-sponsored insurance premium from the stated gross wages, which produced an AGI of $35,750 and a tax of $917.70. Another group subtracted the $9,629 federal overtime-premium deduction, which is taken after AGI and does not reduce it, producing $31,910 and about $764. Other models used the wrong standard deduction ($10,750, $13,000–$21,500), a higher rate (4%–4.75%), or a final number that did not match their own correct calculation."
us,scenario_038,child1_chip_eligible,9,llm_error,categorical_eligibility,False,,"Under PolicyEngine, CHIP only covers children who do not qualify for Medicaid, so CHIP is mutually exclusive with Medicaid. Child 1 is 7, and household income is about $22,992, roughly 70–73% FPL for a family of 4. That is well within Louisiana's Medicaid limit for children aged 6–18, so the engine puts the child in the OLDER_CHILD Medicaid category, and that Medicaid eligibility sets is_chip_eligible to False. Every model that answered checked only whether income was below a CHIP ceiling and marked the child CHIP-eligible. None of them checked first whether the child already qualified for Medicaid."
us,scenario_038,child1_head_start_eligible,3,llm_error,age_disability,False,,"Head Start for preschool-age children requires the child to fall within the program’s preschool age range; a seven-year-old does not satisfy that categorical requirement. Income eligibility cannot override the age restriction, so Child 1 is ineligible even under an income calculation that falls below a poverty guideline."
@@ -341,7 +341,7 @@ us,scenario_038,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,Fal
us,scenario_038,spouse_wic_eligible,5,llm_error,categorical_eligibility,False,,"WIC income eligibility does not independently confer eligibility: a person must also be pregnant, breastfeeding, postpartum, an infant, or a child under age five. The spouse is 18, all unlisted statuses are false, and the only children are ages six and seven, so neither parenthood, childbearing age, nor low household income places the spouse in a WIC categorical group."
us,scenario_038,state_income_tax_before_refundable_credits,10,llm_error,state_local_rule,False,,"Louisiana's 2024 tax reform took effect in 2025. It replaced the graduated brackets and the old combined personal exemption–standard deduction ($9,000 MFJ, plus $1,000 per dependent) with a flat 3% rate and a $25,000 standard deduction for joint filers. Federal AGI is $21,896: $22,993 of wages and SE income minus the half-SE-tax deduction of about $1,097. That is below the $25,000 MFJ deduction, so Louisiana taxable income is zero and so is the tax. Most models used pre-reform exemptions and brackets (1.85%, 2%/4%) or applied only the $12,500 single-filer deduction to a joint return, which left positive taxable income. One model reached $0 in its reasoning but submitted a different number."
us,scenario_038,state_refundable_credits,40,llm_error,state_local_rule,False,,"Louisiana's EITC is refundable and equals 5% of the federal EITC. This MFJ household has two children and earned income of about $21,900 (wages plus SE income minus half of SE tax). That puts it on the federal plateau, so it gets the full 2026 two-child maximum of $7,316, and 5% × $7,316 = $365.80. Most wrong answers fall into one of two groups. The first denied that Louisiana has a refundable EITC, claimed it was nonrefundable, or tied it to state tax liability, and so answered $0. The second used the right 5% structure but a wrong federal EITC figure (an inflated or 2025-level maximum, or a value placed in the phase-in or phase-out range), or applied the wrong state rate (3.5% or 50%)."
-us,scenario_039,federal_income_tax_before_refundable_credits,46,llm_error,household_unit_or_filing_status,False,,"The head is a surviving spouse with no dependent child, so they file as Single. Qualifying-surviving-spouse status needs a qualifying child, so the joint brackets and the $32,200 standard deduction do not apply. Single filing uses the $25k/$34k Social Security thresholds, which makes $30,689.25 of benefits taxable. AGI is $79,209.22, the 2026 single standard deduction is $16,100 (OBBBA), and taxable income is $63,109.22. Tax under the 2026 single brackets (10% to $12,400, 12% to $50,400, 22% above) comes to $8,596.03, with no nonrefundable credits. Most models failed in one of four ways: they applied surviving-spouse joint treatment, assumed the TCJA sunset (personal exemptions and 15%/25% rates), used stale or guessed 2025/2026 deduction and bracket figures, or invented credits that wiped out the tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,47,llm_error,household_unit_or_filing_status,False,,"Qualifying surviving spouse status requires a dependent child. This widowed head has no dependents, so they file single. Correct path: taxable Social Security of $30,689.25 (85% cap), AGI of $79,209.22, the 2026 OBBBA single standard deduction of $16,100 (age 61 gets no add-on), and taxable income of $63,109.22. The 2026 single brackets (10% to $12,400, 12% to $50,400, 22% above) give $8,596.03. About 17 models wrongly used joint-rate surviving-spouse treatment and landed near $4,500–$5,200. Most others used 2025 or guessed 2026 single parameters, assumed the TCJA sunset (15%/25% brackets with personal exemptions), or invented deductions or credits that wiped out the tax. Developer adjudication (2026-11-11): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (household_unit_or_filing_status). The judge labeled eight models' rows reference_later_law (deepseek-v4-pro, gemini-3-flash-preview, gemini-3.1-pro-preview, gemini-3.5-flash, gemini-4-argon, grok-4.5, grok-4.6 and grok-4.7) because they applied the pre-TCJA rate schedule scheduled to return in 2026, with its 15% second bracket, and in most cases a smaller standard deduction and a personal exemption. The reference applies the rate tables, larger standard deduction and zero exemption amount that P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, made permanent, at the 2026 single amounts of Rev. Proc. 2025-32 (a $16,100 standard deduction and the 10%, 12% and 22% brackets), and the benchmark scores law published before the freeze; the rows are model errors, as the 2026-09-29 and 2026-10-14 waves decided for the same rule in scenario_007's and scenario_009's federal income tax. The case keeps the verdict's class: the widowed head has no dependent child, so 26 U.S.C. 2(a)(1)(B) denies surviving-spouse status and the head files single."
us,scenario_039,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_039,head_medicaid_eligible,3,llm_error,health_coverage,False,,"Every Virginia Medicaid pathway open to a 61-year-old is income-tested: the ACA expansion adult group caps MAGI at 138% FPL, and the SSI-linked aged/blind/disabled group sits far below that and keys off SSI receipt, which is $0 here. This head's Medicaid MAGI counts the full $36,105 Social Security benefit (untaxed portion included) plus $26,800 in IRA distributions, $2,030 in pension, and $25,950 in estate income, offset only by a $6,260 self-employment loss, landing at 368% FPL — 2.7x the expansion limit — so the engine assigns medicaid_category NONE. All three models either asserted without arithmetic that MAGI fell under 138% FPL or treated the ""is disabled"" flag as a categorical pass that skips the income test entirely. Disability and the expansion group's waived asset test change which income test applies, never whether one applies."
us,scenario_039,head_medicare_eligible,23,llm_error,age_disability,False,,"PolicyEngine's Medicare eligibility test has two routes: age 65 or older, or positive Social Security disability (SSDI) income with at least 24 months of receipt. A general `is_disabled` flag is not an input to this test. The head is 61 and has no SSDI income or SSDI months (only Social Security retirement income, plus the prompt's zero default for unlisted numeric inputs), so the head fails both routes. Every model treated the disability flag alone, sometimes along with the retirement benefits, as enough for Medicare, and so answered 1 when the answer is 0."
@@ -359,7 +359,7 @@ us,scenario_040,spouse_medicaid_eligible,4,llm_error,categorical_eligibility,Fal
us,scenario_040,spouse_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility applies at age 65, so the 69-year-old spouse qualifies through the age pathway without needing disability-based eligibility or 24 months of SSDI receipt. The spouse’s workers’ compensation and disability facts do not alter that age-based result."
us,scenario_040,state_income_tax_before_refundable_credits,6,llm_error,taxable_income_or_deductions,False,,"The case turns on two Arizona-specific rules: Arizona subtracts from federal AGI all Social Security benefits included there, and Arizona's standard deduction conforms to the base federal married-filing-jointly amount (about $32,200 for 2026) with no additional age-65 amount. Removing the $4,300 of taxable Social Security leaves $28,000 of wages as Arizona AGI, which is more than $4,000 below the standard deduction, so taxable income is zero and the 2.5% flat tax yields nothing before refundable credits; itemizing is worse at $16,324 of mortgage interest plus real estate taxes. The wrong answers all manufacture a positive base either by shrinking the deduction — inventing $20,000, $22,000, or an unstated ~$22,700 figure, in two cases by grafting a federal-style age-65 add-on Arizona does not have — or by leaving Social Security in the Arizona base. One model produced the zero result and then overrode it, and one returned nothing at all."
us,scenario_040,state_refundable_credits,2,llm_error,categorical_eligibility,False,,"Arizona’s refundable credits are not awarded merely because a married filer is elderly and has income below a general threshold. The household satisfies no applicable Arizona refundable-credit eligibility pathway in 2026, so az_refundable_credits and the aggregate state_refundable_credits are both zero; the other failure is a missing required output."
-us,scenario_042,federal_income_tax_before_refundable_credits,51,llm_error,taxable_income_or_deductions,False,,"The head is a widowed filer with no dependent child, so the correct status is single, not qualifying surviving spouse. Under single filing, taxable Social Security is 0.85×(48,525−34,000)+4,500 = $16,846, which gives AGI of $50,580.71. The 2026 OBBBA deductions are $16,100 standard, $2,050 aged add-on and the $6,000 senior deduction, totaling $24,150, so taxable income is $26,430.71. The $4,681 of qualified dividends and capital gains is taxed at 0%, and the $21,749.71 of ordinary income is taxed at 10% up to $12,400 and 12% above, for $2,361.96. The wrong answers fall into four main groups: using QSS joint-width deductions and brackets, leaving out the $6,000 senior deduction, assuming the TCJA expired in 2026 (bringing back personal exemptions and the 15% bracket), and miscalculating how much Social Security is taxable. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The frozen 1,979.16 is regenerated as 2,361.96: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. Reference regenerated (26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7)."
+us,scenario_042,federal_income_tax_before_refundable_credits,52,llm_error,taxable_income_or_deductions,False,,"The head is a widowed 79-year-old with no dependents, so they file Single. Qualifying-surviving-spouse status requires a dependent child. The correct calculation uses the single Social Security thresholds of $25k/$34k, giving $16,846 of taxable benefits and AGI of $50,580.71. Deductions are $24,150: the $16,100 standard deduction, the $2,050 age-65 addition and the $6,000 OBBBA senior deduction. That leaves taxable income of $26,430.71, of which $4,681 is qualified dividends and gains taxed at 0%. The remaining $21,749.71 is taxed at 10% up to $12,400 and 12% above it, giving $2,361.96. The wrong answers fall into three groups: using QSS joint parameters (tax around $600), leaving out the $6,000 senior deduction (around $3,080–3,200), or applying pre-TCJA sunset law with personal exemptions and a 15% bracket (around $3,850–3,960). Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The frozen 1,979.16 is regenerated as 2,361.96: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. Reference regenerated (26 U.S.C. 61(a)(3), 852(b)(3)(B); Form 1040 line 7)."
us,scenario_042,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_042,payroll_tax,2,llm_error,payroll_tax_base,False,,"Payroll tax uses employee wages as its base, and the prompt lists no wage or salary income. The pension, IRA distributions, investment income, Social Security benefits, and farm loss do not generate employee-side Social Security or Medicare tax, so employee payroll tax is $0."
us,scenario_042,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -372,11 +372,11 @@ us,scenario_043,head_medicaid_eligible,10,llm_error,categorical_eligibility,Fals
us,scenario_043,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_043,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_043,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_043,payroll_tax,32,prompt_ambiguity,state_local_rule,False,,"Federal FICA is the easy part: 7.65% of about $4,088 in wages, or about $312.74. The models split on Colorado's FAMLI premium, which PolicyEngine counts as an employee payroll tax: 0.44% (half of the 2026 0.88% premium) on $4,088.15 of FAMLI wages, or $17.99, for a total of $330.73. The prompt asks for 'mandatory employee state payroll taxes' but doesn't say who pays the FAMLI share. Under reading (1), the employee share is withheld from wages and counts. Under reading (2), the employer is allowed to cover it, so it isn't mandatory for the employee and only FICA counts. Most models took reading (2) or didn't know FAMLI exists. A few models used the wrong FAMLI rate, made an arithmetic slip, or wrongly cut the FICA wage base. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 330.73; the alternative is 312.74. The prompt counts mandatory employee state payroll taxes; Colorado FAMLI contributions are the employer's, which may deduct the employee share from wages but need not (C.R.S. 8-13.3-507(2) and (5); 7 CCR 1107-1 s. 1.4(6)(A)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($17.99); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
+us,scenario_043,payroll_tax,33,prompt_ambiguity,state_local_rule,False,,"The reference has three parts: employee FICA on $4,088 of wages ($253.47 Social Security plus $59.28 Medicare), plus Colorado's FAMLI employee premium of $17.99. The FAMLI premium is 0.44% of wages, half of the 0.88% total premium for 2026. Colorado lets the employer pay the employee's half, and the prompt never says who pays it. So the question supports two readings: either the 0.44% is withheld from wages (the reference, $330.73), or the employer covers it and only FICA applies ($312.74). Most models chose the FICA-only reading. A few models made separate errors: one used the wrong FAMLI rate, one submitted a number that doesn't match its own math, and two zeroed out FICA wages. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 330.73; the alternative is 312.74. The prompt counts mandatory employee state payroll taxes; Colorado FAMLI contributions are the employer's, which may deduct the employee share from wages but need not (C.R.S. 8-13.3-507(2) and (5); 7 CCR 1107-1 s. 1.4(6)(A)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($17.99); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
us,scenario_043,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_043,snap,35,llm_error,thresholds_rates,False,,"This one-person Colorado household has an elderly 66-year-old head. It has $387 a month of gross income and $2,800 in assets, so it passes the elderly asset test and is categorically eligible through Colorado's broad-based TANF non-cash eligibility. Its deductions bring countable net income to zero, so the benefit equals the full one-person maximum allotment for the contiguous states: $298 a month, or $3,576 a year. Most models correctly found that the household gets the maximum allotment but used an outdated or invented figure ($291, $292, $299–$305, $328). A smaller group wrongly zeroed the benefit by applying a non-elderly asset limit or ignoring take-up, and a few arbitrarily cut the allotment below the maximum even though net income was zero."
+us,scenario_043,snap,36,llm_error,thresholds_rates,False,,"The 66-year-old head makes this an elderly household. It is categorically eligible through Colorado's broad-based categorical eligibility (BBCE), and the $2,800 of assets is under the elderly limit. Monthly gross income is about $387. Subtracting the 20% earned-income deduction, the $209 standard deduction and the excess medical deduction ($5,000/12 − $35 ≈ $382) brings net income to $0, so every month pays the one-person contiguous-states maximum allotment of $298, or $3,576 a year. Most models got net income to zero but used the wrong maximum allotment: $291 (FY2024), $292 (FY2025), a guessed uprated figure, or an arbitrary sub-maximum amount. A smaller group wrongly zeroed the benefit with asset or income tests."
us,scenario_043,ssi,4,llm_error,period_annualization,False,,"SSI eligibility is determined for the individual, not by subtracting annualized countable income from twelve times a monthly federal benefit rate. The head’s approximately $4,642 of individual income exceeds the applicable SSI income limit, so the income-eligibility gate fails and no SSI is payable despite aged categorical eligibility and assumed take-up."
-us,scenario_043,state_refundable_credits,7,llm_error,state_local_rule,False,,"Colorado's refundable credits available to this filer depend on two things. The TABOR surplus mechanisms (the sales tax refund and the senior housing credit) pay only when the state has excess revenue. The Colorado EITC is a percentage of the federal EITC. Forecasts showed FY2025-26 below the Referendum C cap, so no TABOR-funded refund exists for tax year 2026. The head is 66, above the federal childless-EITC age ceiling of 64, so there is no federal EITC for Colorado to match. Every wrong answer either paid a TABOR-contingent refund that is not triggered in 2026 or matched a federal EITC the head cannot claim because of age."
+us,scenario_043,state_refundable_credits,8,llm_error,state_local_rule,False,,"Colorado's TABOR sales tax refund under C.R.S. 39-22-2003(2) is paid only when the state has excess revenue in the fiscal year ending in the tax year. Forecasts put FY2025-26 below the Referendum C cap, so there is no tax year 2026 refund. Colorado's EITC is a percentage match of the federal EITC, and the federal childless EITC requires an age of 25–64, so the 66-year-old head gets $0 federal EITC and $0 Colorado EITC. No Colorado senior housing or senior-specific refundable credit pays this household for 2026. The models either carried forward an earlier year's TABOR refund, applied an EITC match without the age bar, or invented a senior credit."
us,scenario_043,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_044,federal_income_tax_before_refundable_credits,5,llm_error,taxable_income_or_deductions,False,,"The couple's only income is $56,806 of Social Security. Their provisional income is 50% of benefits plus other income: 0.5 × $56,806 + $0 = $28,403. That is below the $32,000 married-filing-jointly base amount under IRC §86, so none of the benefits are taxable. AGI and taxable income are $0, and so is regular income tax. Every wrong model skipped this test. They treated the benefits as taxable in full, assumed the 85% maximum inclusion applied automatically, or wrongly said the threshold was exceeded. Each then put a positive tax on income that is excluded by statute."
us,scenario_044,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages and other covered earned compensation, not Social Security retirement benefits. Because the household has no reported wages or self-employment income, every employee payroll-tax component has a zero base and payroll tax is $0."
@@ -411,7 +411,7 @@ us,scenario_048,head_medicare_eligible,1,llm_error,other,False,,"Medicare eligib
us,scenario_048,payroll_tax,2,llm_error,other,False,,"Payroll tax here is just employee FICA on $7,656 of wages: 6.2% Social Security ($474.67) plus 1.45% Medicare ($111.01), for $585.68. Ohio has no mandatory employee state payroll tax. Social Security survivor benefits are not covered wages. Both models arrived at about $586 in their reasoning but then put a different, unsupported number in the numeric value field. The failure is in transcribing the answer, not in the tax rule."
us,scenario_048,snap,5,llm_error,thresholds_rates,False,,"The household’s $34,626 annual income exceeds the applicable $17,420 SNAP eligibility threshold for a one-person Ohio household in 2026, so SNAP eligibility ends before any minimum-allotment calculation. A minimum benefit applies only to an otherwise eligible household; it cannot override a failed income test."
us,scenario_048,state_income_tax_before_refundable_credits,2,llm_error,thresholds_rates,False,,"The trap is Ohio's zero-tax bracket. Ohio's individual income tax schedule imposes no tax on Ohio taxable nonbusiness income at or below roughly $26,050, so this filer's $7,656 in wages — the only Ohio-taxable income, since the $26,970 in Social Security survivor benefits stays out of federal AGI at a provisional income of $21,141, below the $25,000 base amount — produces $0 of liability before any credit. Both models correctly excluded the Social Security benefits and then applied the 2.75% rate starting at the first dollar of taxable income, converting a below-threshold filer into a taxpayer. Their differing answers ($106 vs. $94.54) come only from downstream inventions — a fictitious Ohio standard deduction in one case, a recharacterized exemption credit plus the $50 senior citizen credit in the other — layered on the same missing zero bracket."
-us,scenario_049,federal_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"The reference sets AGI at $247,385: wages minus the $15,436 traditional 401(k), plus $3,501 of taxable interest and $6,320 of dividends, with no exclusion for ESI premiums and no deduction for the IRA, which phases out. From that it subtracts three OBBBA-era deductions: the $32,200 2026 MFJ standard deduction, the full $25,000 MFJ qualified-overtime deduction (MAGI is below the $300,000 phase-out start), and the roughly $580 non-itemizer cash charitable deduction. The $928 auto-loan-interest deduction is zero because it is fully phased out above $200,000 MAGI. Most models dropped the $25,000 overtime deduction or applied TCJA-sunset rules, which overstates tax by about $5,500 or more. The models that came close went wrong on smaller items: excluding ESI premiums, claiming the phased-out auto-loan interest, or leaving out the non-itemizer charitable deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,48,llm_error,taxable_income_or_deductions,False,,"This answer depends on 2026 law after the One Big Beautiful Bill Act (OBBBA), which made the TCJA brackets and higher standard deduction permanent. AGI is $247,385: wages are reduced only by the $15,436 traditional 401(k) deferral, and the IRA deduction is zero because the spouse is an active plan participant above the phaseout. Taxable income then subtracts the $32,200 standard deduction, the $25,000 joint qualified-overtime deduction (not phased out because MAGI is under $300,000), and the $579.76 non-itemizer cash-charity deduction. The auto-loan interest deduction phases out completely at this MAGI. Most models missed the overtime deduction, applied TCJA-sunset rules with personal exemptions, or misstated AGI by excluding employer insurance premiums, deducting the IRA contribution, or claiming the auto-loan interest."
us,scenario_049,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_049,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_049,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -436,18 +436,18 @@ us,scenario_051,federal_refundable_credits,4,llm_error,thresholds_rates,False,,"
us,scenario_051,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Louisiana Medicaid eligibility requires qualification through a specific categorical pathway, not merely acceptable immigration status or the absence of an expressly listed disqualification. This 25-year-old, nondependent adult has MAGI income of 2.51 times FPL, receives no SSI, and falls into Medicaid category NONE, so no MAGI or alternative state-program pathway confers eligibility."
us,scenario_051,payroll_tax,10,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes apply to the full $40,000 of wages here, producing $2,480 and $580 respectively, for a $3,060 total. The dominant error was subtracting the listed employer-sponsored insurance premium without any fact establishing a pre-tax salary-reduction arrangement; several other models derived $3,060 correctly but then submitted a different number."
us,scenario_051,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_051,state_income_tax_before_refundable_credits,51,reference_later_law,taxable_income_or_deductions,False,,"Louisiana's post-2024 reform (Act 11) uses a flat 3% rate, starts from federal AGI of $40,180, and allows a single standard deduction of $12,500 that is indexed for inflation from 2026. The retained reference is $820.35 using the earlier computed $12,835 deduction. RIB 26-019 published $12,838 after the freeze, giving $820.26; this output is excluded under d994. Most models used the unindexed $12,500 and got $830.40. Others used the repealed pre-2025 system: the 1.85%/3.5% brackets, the $4,500 exemption, and the deduction for federal income tax paid. Some also subtracted employer insurance premiums from AGI or answered that Louisiana has no income tax. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_later_law (taxable_income_or_deductions). The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. The frozen reference is 820.35; the value under the published figure is 820.26. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana). Reference depends on law published after the reference freeze; output excluded from scoring (Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published)."
+us,scenario_051,state_income_tax_before_refundable_credits,52,reference_later_law,taxable_income_or_deductions,False,,"Louisiana's 2024 reform (Act 11, 3rd Extraordinary Session) created a flat 3% rate and a $12,500 single standard deduction that is indexed for inflation starting in 2026. For 2026 the deduction is $12,835, so taxable income is $40,180 − $12,835 = $27,345 and the tax is $820.35. Most models used the unindexed $12,500 figure and got $830.40. The rest used the repealed pre-2025 structure: the $4,500 exemption, 1.85%/3.5% brackets, and the federal income tax deduction. Some also subtracted the employer insurance premiums from the given gross wages, used the federal standard deduction, or said Louisiana has no income tax. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_later_law (taxable_income_or_deductions). The frozen 814.95 is regenerated as 820.35: it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#8839 (issue #8828), after the reference freeze): Capital gain distributions are long-term capital gain in gross income and in the preferential-rate base. The frozen reference is 820.35; the value under the published figure is 820.26. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana). Reference depends on law published after the reference freeze; output excluded from scoring (Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published)."
us,scenario_051,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_052,federal_income_tax_before_refundable_credits,50,reference_engine_defect,taxable_income_or_deductions,False,"PolicyEngine deducts the $43.28 traditional IRA contribution above the line without applying the IRC §219(g) active-participant phaseout. The 2026 MFJ phaseout runs $129,000–$149,000, and the head makes 401(k) elective deferrals with MAGI of about $504,800, so the deduction is overstated by $43.28 and tax is understated by $13.89.","The frozen, unscored reference comprises ordinary tax of $103,865.78 on $471,730.56, plus $127.20 of 15% tax on the $848 of qualified dividends, plus $218.42 of NIIT on $5,748 of net investment income. That ordinary amount comes from taxable income of $472,578.56: AGI of $504,778.56 (after the $926.16 401(k) deferral, the $3,000 capital-loss cap and the $43.28 IRA deduction) less the $32,200 OBBBA standard deduction, taxed under the 2026 MFJ brackets. Common differences from the frozen reference involve TCJA-sunset parameters, ESI premiums subtracted from wages, differing NIIT treatment and the $2,250 Additional Medicare Tax assigned to payroll tax. The exclusion record leaves NIIT's inclusion in the requested output unresolved. The three closest answers ($104,225.00 and two at $104,225.30) disallowed the IRA deduction and fall within $0.26 of the recorded corrected value, $104,225.26; their differences from the frozen reference also include rounding. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 104,211.41; the corrected value is 104,225.26. The frozen reference deducted traditional IRA contributions without the active-participant phase-out. The policyengine-us 2.60.0 engine applies that phaseout; the output remains excluded because NIIT's inclusion in the requested output remains unresolved. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
us,scenario_052,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_052,head_medicare_eligible,1,llm_error,age_disability,False,,"Standard Medicare eligibility requires age 65 unless a separate qualifying pathway, such as disability or end-stage renal disease, applies. The head is 59 and all unlisted disability and health-status facts are false, so no Medicare eligibility pathway applies."
us,scenario_052,payroll_tax,32,llm_error,payroll_tax_base,False,,"Only the head earns wages. The reference applies three components to the full $500,000: 6.2% Social Security on the 2026 wage base of $184,500 ($11,439), 1.45% Medicare on all $500,000 ($7,250), and 0.9% Additional Medicare Tax above the $250,000 married-filing-jointly threshold ($2,250), for $20,939. Two errors account for most misses. Some models used an outdated or guessed Social Security wage base, such as $168,600, $176,100 or about $180,000. Others shrank Medicare wages by subtracting the employer-sponsored insurance premium or the traditional 401(k) deferral, even though the facts never describe a cafeteria plan and 401(k) deferrals stay subject to FICA. Other models used the $200,000 single threshold, made arithmetic slips, or submitted numbers that did not match their own work."
us,scenario_052,spouse_medicare_eligible,1,llm_error,age_disability,False,,"Medicare eligibility requires age 65 or an applicable disability or other qualifying pathway. The 55-year-old spouse has no listed disability or other Medicare-qualifying status, so the spouse is not eligible."
-us,scenario_053,federal_income_tax_before_refundable_credits,38,llm_error,thresholds_rates,False,,"AGI is the full $66,968.68 of wages. There are no above-the-line adjustments, and the $950 of auto loan interest gives no deduction. The 2026 OBBBA standard deduction of $16,100 leaves taxable income of $50,868.68. Under the permanent 2026 single brackets, tax is 10% up to $12,400 plus 12% up to $50,400 plus 22% on the last $468.68, which totals $5,903.11. No nonrefundable credits apply. The wrong answers come from four errors: assuming the TCJA expired, using stale 2023–2025 deductions or brackets, subtracting the ESI premiums or auto loan interest from income, or working out about $5,903 and then submitting a different number."
+us,scenario_053,federal_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"The reference uses full wages of $66,968.68 as AGI, with no exclusion for ESI premiums and no auto-loan-interest deduction. It then subtracts the 2026 standard deduction of $16,100 (made permanent by the OBBBA) to get taxable income of $50,868.68. The 2026 single brackets apply: 10% up to $12,400 ($1,240), 12% up to $50,400 ($4,560), and 22% on the remaining $468.68 ($103.11), for $5,903.11 with no nonrefundable credits. The wrong answers fall into four groups. Some models assumed the TCJA expired, using an $8,300 standard deduction, a personal exemption, and 15%/25% rates. Some used 2024 or 2025 deductions and brackets. Some subtracted the $3,389 ESI premiums or the $950 auto-loan interest even though the facts don't support either. Others computed the right figure but submitted a different number."
us,scenario_053,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"A single 25-year-old filer with no children and $66,969 of wages receives neither a refundable Child Tax Credit nor an Earned Income Tax Credit in 2026. With the other enumerated refundable-credit components also equal to zero, federal refundable credits total $0."
us,scenario_053,payroll_tax,10,llm_error,payroll_tax_base,False,,"PolicyEngine applies the employee FICA rates to the full $66,969 of gross wages: 6.2% Social Security ($4,152.06) plus 1.45% Medicare ($971.05) equals $5,123.10. The listed employer-sponsored insurance premium ($3,389) does not reduce the payroll-tax wage base. The household facts never say the premium is paid through a Section 125 cafeteria plan, and the prompt forbids inferring unlisted facts. Eight models subtracted the premium anyway and taxed $63,580, which gives $4,863.87. The two GPT-5.4 models used the correct base and rates but multiplied them out wrong."
us,scenario_053,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_053,state_income_tax_before_refundable_credits,51,llm_error,state_local_rule,False,,"Idaho's 2026 tax is a flat 5.3% on taxable income above a $4,811 zero-rate bracket. To reach the reference, start from federal AGI of $66,969, which the ESI premiums do not reduce. Subtract the $5,000 Idaho subtraction for health insurance premiums the household pays and the $16,100 standard deduction. That leaves $45,869 of taxable income, and the tax on it is $2,176.06. Then add the $10 permanent building fund tax to get $2,186.06. Most models used the repealed 5.695%/5.8% rate, skipped the zero bracket, missed the $5,000 health-premium subtraction, or left out the $10 building fund tax."
+us,scenario_053,state_income_tax_before_refundable_credits,52,llm_error,state_local_rule,False,,"Idaho's 2026 tax uses three state-specific steps. First, Idaho subtracts the $5,000 of health insurance premiums the household pays itself, so Idaho AGI is $61,968.68. Wages are not reduced by the $3,389 ESI premium. Second, after the $16,100 standard deduction, Idaho charges 5.3% only on taxable income above the $4,811 single zero-rate amount: ($45,868.68 − $4,811) × 5.3% = $2,176.06. Third, the $10 permanent building fund tax is added, giving $2,186.06. Most models taxed $50,869 at a flat 5.3% or 5.695% with no zero bracket. Almost every model missed the health-premium subtraction and the $10 permanent building fund tax."
us,scenario_053,state_refundable_credits,43,llm_error,categorical_eligibility,False,,"Idaho's grocery credit is refundable, with no income limit or phase-out. Its amount is prorated by months without SNAP receipt or incarceration. Idaho's 2025 legislation (HB 231) raised it to $155 per person under age 65 starting in 2025, so this single 25-year-old who qualified all 12 months gets $155. Of the wrong models, 21 answered $0, 21 used a per-person amount of $100, $120, or $150, and one provided no output."
us,scenario_054,child1_chip_eligible,7,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child to be ineligible for Medicaid; satisfying a CHIP income ceiling alone is not sufficient. At this very low household income, the 10-year-old qualifies for Medicaid under North Carolina’s OLDER_CHILD category, and that Medicaid eligibility excludes the child from CHIP."
us,scenario_054,child1_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Child 1 qualifies through the ordinary children's MAGI pathway: at age 10 he falls in the OLDER_CHILD category (ages 6–18), whose North Carolina income limit is 133% FPL plus the 5-percentage-point disregard, and NC folded its separate NC Health Choice tier into Medicaid so school-age children are covered under Medicaid rather than being pushed to CHIP. The household's only income is $17,100 of self-employment earnings, which after the deductible half of SE tax leaves MAGI at 0.58 × FPL for a family of three — a fraction of the applicable limit. Both models returned 0 without ever performing that categorical-plus-income test: one asserted an income failure that the arithmetic contradicts, the other treated unlisted household facts as missing prerequisites even though the prompt instructs that unlisted booleans are false and take-up is assumed."
@@ -465,7 +465,7 @@ us,scenario_054,head_wic_eligible,17,llm_error,categorical_eligibility,False,,"W
us,scenario_054,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wages, not SSTB self-employment income; that income instead enters the separate self-employment-tax calculation. With no reported wages and all unlisted numeric inputs set to zero, every component of the requested employee-side payroll tax is zero."
us,scenario_054,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier in this output. The household is in the FREE tier because it meets categorical eligibility through SNAP or TANF and its school-meal income is 63% of the federal poverty guideline, below the 130% free-meals threshold; therefore reduced-price support is zero."
us,scenario_054,self_employment_tax,7,llm_error,other,False,,"This is a plain SECA calculation. Net SE earnings are $17,100 × 0.9235 = $15,791.85, which is far below the Social Security wage base, so the full 15.3% applies: $15,791.85 × 0.153 = $2,416.15. Every model either picked the right method and then made an arithmetic or rounding slip, or worked out about $2,416 and then submitted some other number. Two models subtracted things that do not reduce SE tax at all, such as the deduction for half of SE tax, which only lowers income-tax AGI."
-us,scenario_054,snap,49,llm_error,thresholds_rates,False,,"The household is categorically eligible through North Carolina's TANF non-cash BBCE, so the $14,000 bank balance does not disqualify the household. Gross self-employment income of $1,425/month is counted with no SE-tax or cost-of-business reduction. The $209 standard deduction and $285 earned income deduction leave $931 net. The 30% expected contribution of $279.30 rounds up to $280, so each month is $785 − $280 = $505, or $6,060 a year. Wrong answers fall into three groups: models that wrongly applied an asset or income cutoff and returned $0; models that used stale or guessed FY2025/FY2026 maximum allotments and standard deductions ($766/$768/$800+, $198/$204/$228); and models that had every input right but skipped the rule that rounds the 30% contribution up to the next whole dollar, which gave $505.70/month ($6,068.40)."
+us,scenario_054,snap,50,llm_error,thresholds_rates,False,,"The correct path counts the full $1,425 a month of self-employment income as gross earned income, with no SE-tax or business-expense deduction. It subtracts the $285 earned income deduction (20%) and the $209 standard deduction to reach net income of $931. Thirty percent of that is $279.30, which rounds up to a $280 expected contribution. The $785 maximum allotment minus $280 gives $505 a month, or $6,060 a year. TANF non-cash categorical eligibility (BBCE) waives the asset test, so the $14,000 in the bank does not disqualify the household. The wrong answers fall into three groups: about ten models used the right inputs but did not round the contribution up, giving $505.70 or $506 a month; about twenty used stale or guessed maximum allotments and standard deductions; and six applied the waived asset test and returned $0."
us,scenario_054,state_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"Two independent paths lead to $0 of North Carolina tax. First, the only income fact listed is the SSTB *component* input; total self-employment income is unlisted and therefore 0 under the prompt's ""treat any unlisted numeric input as 0"" rule, so the engine's trace shows essentially no income entering nc_income_tax. Second, even crediting the full $17,100 as net self-employment earnings, federal AGI is $17,100 − $1,208 (half of SE tax) = $15,892, and North Carolina's 2026 head-of-household standard deduction of $19,125 — the head has two qualifying children — exceeds that, driving NC taxable income to $0 and the 3.99% 2026 flat rate onto a zero base. Every wrong model produced a positive liability; two of them derived exactly $0 in their own reasoning and then submitted a different number anyway."
us,scenario_055,federal_income_tax_before_refundable_credits,40,llm_error,taxable_income_or_deductions,False,,"AGI is $156,275.86, with Social Security at the 85% maximum ($4,880.71). Under OBBBA law for 2026, the filer gets a $16,100 basic standard deduction, a $2,050 aged add-on, a $6,000 senior deduction cut by 6% of MAGI above $75,000 (leaving $1,123.45), and the new $1,000 charitable deduction for people who don't itemize. Together these come to $20,273.45, which leaves $136,002.41 of taxable income. Tax on that amount under the 2026 brackets (12,400/50,400/105,700) is $25,238.58. The closest models ($25,479) left out only the $1,000 charitable deduction for non-itemizers. Many others assumed the TCJA expired after 2025 and used personal exemptions with the old 15/25/28% brackets, or they skipped the phased senior deduction or applied the full $6,000."
us,scenario_055,payroll_tax,5,llm_error,thresholds_rates,False,,"The only FICA-taxable income here is the $83,635 of wages: the $67,760 taxable pension and $5,742 Social Security retirement benefit are outside the payroll tax base, the wage is far below the 2026 Social Security wage base, age 70 does not exempt wages from FICA, and Florida levies no mandatory employee state payroll tax. The answer is therefore exactly 6.2% × $83,635 = $5,185.37 plus 1.45% × $83,635 = $1,212.71, or $6,398.08. What separates the reference from the wrong answers is execution of that single multiplication at the employee-side rate: two models stated the right rule and then mis-multiplied or rounded, one submitted a number contradicting its own correct explanation, one used the combined 12.4%/2.9% employer-plus-employee rates before guessing, and one submitted a placeholder zero."
@@ -475,22 +475,22 @@ us,scenario_056,head_wic_eligible,1,parse_contract_failure,missing_output,False,
us,scenario_056,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_056,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wage earnings, and the household has no reported wages or salary. The listed SSTB self-employment income belongs in the separately requested self-employment-tax calculation, while alimony and financial assistance are not employee FICA wages."
us,scenario_056,self_employment_tax,7,llm_error,payroll_tax_base,False,,"The calculation is $4,340 SSTB self-employment income × 92.35% = $4,007.99 in net SE earnings. That is above the $400 filing floor and well below the Social Security wage base, so the full 15.3% applies, giving $613.22. The models went wrong in three ways. Some dropped or misread the SE income and said it fell below the $400 threshold. Some applied roughly half the combined rate. One came within about $1 but rounded or computed loosely. The alimony, financial assistance, HSA deduction and itemized deductions do not change the SE tax base."
-us,scenario_056,snap,50,prompt_ambiguity,categorical_eligibility,False,"PolicyEngine's SNAP shelter cost leaves out the home_mortgage_interest input. The excess shelter deduction therefore uses only the $995.75/month in real estate taxes and drops the $564/month mortgage interest, although 7 CFR 273.9(d)(6)(ii)(A) counts mortgage payments, including interest, as shelter costs. Counting the interest would hit the shelter cap and produce about $165–170/month instead of the $95 New Jersey minimum.","The $1,140 annual reference is frozen and excluded because work hours and occupied-home mortgage status are unlisted. Its historical derivation assumed 40 weekly work hours and property taxes alone as shelter costs, producing a $63.18 formula benefit raised to New Jersey's $95 monthly minimum. The occupied-home reading counts mortgage interest as a shelter cost. The sidecar's 2.60.0 recheck records $95 annually and preserves the exclusion; its earlier recheck reproduces $1,908 annually with 40 weekly hours and occupied-home mortgage interest. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies)."
+us,scenario_056,snap,51,prompt_ambiguity,categorical_eligibility,False,,"PolicyEngine counts all $1,486.54/month of income, including the $1,000 of financial assistance. It deducts $72.33 for earned income and $213.68 as the standard deduction. Shelter cost comes only from the $995.75/month in real estate taxes; the home mortgage interest input is not a SNAP shelter expense. That gives an uncapped $395.49 excess shelter deduction and $805.03 of net income. The federal formula ($304.68 − $241.50) yields only $63.18, and New Jersey's state minimum benefit raises that to $95/month, or $1,140 a year. Models that added mortgage interest to shelter costs hit the shelter cap and got about $140–160/month. Models that answered $0 either misapplied the income, asset, or work tests, or skipped the minimum benefit owed to an eligible one-person household. Two models applied the federal $23 minimum instead of New Jersey's $95. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 1,140.00; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. The prompt also lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr and whether the listed home mortgage interest is on the home the SNAP household occupies)."
us,scenario_056,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_056,state_income_tax_before_refundable_credits,5,llm_error,thresholds_rates,False,,"New Jersey's gross income tax exempts a single filer entirely when New Jersey gross income falls at or below the $10,000 minimum filing threshold, and this household's NJ gross income is $5,838.53 (alimony $1,498.53 plus self-employment $4,340.00; the $12,000 of financial assistance is not NJ gross income). PolicyEngine therefore computes the bracket tax as an intermediate figure — $4,838.53 of taxable income after the $1,000 regular exemption, times the 1.4% first-bracket rate, equals $67.74 — and then zeroes it out under the below-threshold exemption, leaving $0.00. Every wrong model stopped at the 1.4% bracket arithmetic and never applied the threshold exemption; three of them also mis-specified the NJ income and deduction base along the way."
-us,scenario_056,state_refundable_credits,51,llm_error,credit_phaseout,False,"PolicyEngine's NJ path for childless filers aged 18–24 caps the phased-in federal-equivalent credit ($308.55) at 40% of the 2026 federal childless maximum (0.40 × $664 = $265.60) and never applies the 40% match to the phased-in amount. N.J.S.A. 54A:4-7 sets the NJ EITC at 40% of the federal-equivalent credit, which gives 0.40 × $308.55 = $123.42.","New Jersey lets childless filers claim its EITC from age 18, not from the federal minimum age of 25. That makes this 20-year-old eligible. Countable earnings are $4,340 of SE income minus half of SE tax ($306.61), or $4,033.39. The engine uses the $308.55 federal phased-in amount to test income eligibility and pays 40% of the $664 federal childless maximum for 2026, or $265.60. Most models used the federal 25–64 age rule and answered $0. A few computed 40% × $308.55 = $123.42. Others used gross SE income ($132.80) or claimed the $50 property-tax credit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated llm_error (credit_phaseout). New Jersey bases the earned income credit for filers 18 and older who fail only the federal age test on the federal maximum credit for filers without a qualifying child, and pays it as a flat yearly amount: 40% of the 2026 $664 maximum is $265.60. The judge's phase-in reading fails. Reference affirmed (N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32)."
+us,scenario_056,state_refundable_credits,52,llm_error,credit_phaseout,False,"The reference of $265.60 is exactly 40% of the 2026 maximum childless EITC of $664. That means PolicyEngine's New Jersey young-childless EITC path used the maximum credit (less a zero phase-out) instead of capping the credit at the phased-in amount, 7.65% × $4,033.39 = $308.55. This conflicts with IRC §32(a)(1), which N.J.S.A. 54A:4-7 incorporates through the 40% match, so the statutory value is $123.42.","The common mistake was tying New Jersey's EITC to federal eligibility. The federal childless credit starts at age 25, but New Jersey pays its 40% match to childless filers aged 18 and older, so this 20-year-old with $4,033.39 of net self-employment earnings qualifies. Most models stopped at the federal age rule and answered $0. The models that applied the expansion computed 40% × 7.65% × $4,033.39 = $123.42. The $265.60 target equals 40% of the full $664 maximum childless credit, which none of the phase-in-based calculations reach. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated llm_error (credit_phaseout). New Jersey bases the earned income credit for filers 18 and older who fail only the federal age test on the federal maximum credit for filers without a qualifying child, and pays it as a flat yearly amount: 40% of the 2026 $664 maximum is $265.60. The judge's phase-in reading fails. Reference affirmed (N.J.S.A. 54A:4-7(a)(4); Rev. Proc. 2025-32)."
us,scenario_056,tanf,2,llm_error,categorical_eligibility,False,,"TANF is categorically limited to assistance units containing a dependent child: New Jersey's WorkFirst NJ/TANF requires a needy child under 18 (or 18 and a full-time secondary student) in the unit, and childless adults are served instead by WFNJ General Assistance, which is not TANF. This household is a single 20-year-old head with no children, so the benefit is $0 regardless of the income test. The second trap is that `financial_assistance` is an input income line in the household facts, not a computed program output; echoing it back as the TANF benefit both invents categorical eligibility and confuses an input with the requested computed value."
us,scenario_057,federal_refundable_credits,37,llm_error,thresholds_rates,False,,"The whole $664 is the no-qualifying-child EITC. Head's $10,400 of self-employment income, after the deduction for half of SE tax (about $9,665), counts as earned income. That is above the 2026 earned-income amount of about $8,680 and well below the joint-filer phase-out start of about $18,000, so the couple gets the full 2026 childless maximum of $664. For a joint return, the 25–64 age test needs only one spouse to qualify, and Head (43) does, so the 20-year-old spouse does not block the credit. Most wrong answers either used a stale or misindexed maximum ($620–$667), placed the couple in the phase-in or phase-out range by mistake, or wrongly denied eligibility over the spouse's age or over self-employment income not counting as earned income."
us,scenario_057,head_medicaid_eligible,4,llm_error,categorical_eligibility,False,,"The head is blind and disabled, with very low countable income ($10,400 of self-employment income, $14 of disability benefits) and only $912 in assets. That makes the head an SSI recipient, receiving about $7,231 a year. Louisiana gives automatic Medicaid to SSI recipients, so the engine puts the head in the SSI_RECIPIENT category. The head would also qualify under Louisiana's adult expansion, because household MAGI is only 0.45× FPL, far below the 138% limit. All four models missed the SSI pathway, and none of them checked the head's income against the expansion limit correctly."
us,scenario_057,head_medicare_eligible,10,llm_error,age_disability,False,,"Before 65, a person qualifies for Medicare only after 24 months of Social Security disability (SSDI) entitlement, or through the ESRD/ALS pathways. PolicyEngine checks this with months of Social Security disability receipt, and it does not use the is_disabled or is_blind flags. The head is 43. Their only disability income is $14 of generic ""disability benefits"", which is not Social Security disability. No months of SSDI receipt are listed, so that input is 0, and neither ESRD nor ALS is listed either. Every model treated disability status plus a nominal disability-benefit amount as enough for Medicare under 65. That skips both the SSDI-entitlement requirement and the 24-month waiting period."
us,scenario_057,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_057,self_employment_tax,7,llm_error,thresholds_rates,False,,"The whole case reduces to one canonical computation: net earnings from self-employment are 92.35% of the $10,400 SSTB profit ($9,604.40), and the combined OASDI/HI rate of 15.3% applies in full because $9,604.40 is far under the 2026 Social Security wage base, giving 0.153 × 9,604.40 = $1,469.47. The $400 figure in §1402(b) is the floor at which SE tax begins, not a ceiling below which it is waived, and no other floor exists; the one-half-of-SE-tax figure is an above-the-line deduction under §164(f), not the liability. The prompt further defines this output as SE tax excluding Additional Medicare Tax, and the 0.9% AMT threshold for joint filers is $250,000 of combined earnings, so nothing is added on top. Every wrong answer comes from inverting the $400 floor, inventing a different floor, truncating the 0.9235 base factor, reporting the deduction instead of the tax, or bolting on a nonexistent AMT."
-us,scenario_057,snap,51,prompt_ambiguity,age_disability,False,"The reference's $602.58/month SSI equals the 2026 individual FBR ($994) minus only the blind head's countable income ($391.42), with $0 for the disabled, married 20-year-old spouse. That treats the spouse as SSI-ineligible, which conflicts with 42 U.S.C. 1382c(a)(3) and the couple benefit rate in 20 CFR 416.412: with a 2026 couple FBR of $1,491, two eligible spouses would get about $1,098.58/month, and that would raise SNAP countable income.","The reference counts three kinds of SNAP income: $866.67/month of self-employment earnings, $3.17/month of disability benefits, and $602.58/month of SSI paid to the head alone at the individual rate. Gross income is $1,472.42. Under the head-only SSI reading, monthly net income rounds to $1,090 and the benefit is $219/month throughout 2026, totaling $2,628 under the held FY2026 schedule. The published $2,669.22 remains a frozen excluded value. Models that left SSI out of SNAP income landed near $4,600–4,800. Models that counted couple-rate SSI of about $1,099/month landed near $840. The output is excluded from scoring for every model because the spouse's SSI disability criterion is unstated. The recorded alternative SSI reading yields $840 annually. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 2669.217041 to 883.617065 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Moves with the additional SSI counted as household income. The 2026-09-22 audit recomputes the alternative as 840.00 with every publication convention and upstream fix (the frozen-engine value above was 883.62). Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)."
+us,scenario_057,snap,52,prompt_ambiguity,age_disability,False,"The derivation's SSI of $602.58/mo equals the $994 individual FBR minus only the head's countable income ($391.42), so PolicyEngine gave the disabled, married spouse no SSI. Under 42 U.S.C. 1382(b)(2) and 20 CFR 416.412, two disabled spouses living together form an eligible couple, which yields $1,491 − $392.42 = $1,098.58/mo; using that figure, SNAP gross income is understated by about $496/mo.","What separates the answers is how much SSI goes into SNAP gross income. The reference counts $602.58/mo of SSI (the $994 individual FBR minus the head's $391.42 countable income), plus $866.67 of self-employment earnings and $3.17 of disability benefits, for a gross of $1,472.42. After the $173.33 earned-income deduction and the $209 standard deduction, net income is $1,090.08, giving $546 − $327 = $219/mo for Jan–Sep and $232.74/mo for Oct–Dec, or $2,669.22 a year. Models that counted the eligible-couple SSI (about $1,098.58/mo) landed near $840. Models that left SSI out landed near $4,700. The rest denied eligibility outright, even though net income is only 62% of the poverty line and assets are $912. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 2669.217041 to 883.617065 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Moves with the additional SSI counted as household income. The 2026-09-22 audit recomputes the alternative as 840.00 with every publication convention and upstream fix (the frozen-engine value above was 883.62). Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)."
us,scenario_057,spouse_chip_eligible,1,llm_error,categorical_eligibility,False,,"CHIP requires the person to be outside Medicaid eligibility; a low income alone does not establish CHIP eligibility. The spouse is Medicaid-eligible in Louisiana's adult category, so the Medicaid exclusion makes `spouse_chip_eligible` false without applying a CHIP income threshold."
us,scenario_057,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Louisiana adopted the ACA adult expansion effective July 1, 2016, so any non-elderly adult aged 19–64 who is not otherwise eligible qualifies for Medicaid on income alone at or below 138% FPL under MAGI methodology — no pregnancy, parenthood, or disability category is required. The spouse is 20 with household MAGI at 0.45 x FPL (the only income is the head's $10,400 SSTB self-employment plus $38 of disability benefits, against a two-person 2026 FPL near $21,150 and a 138% cutoff near $29,200), placing the household roughly a third of the way to the expansion limit. Every wrong model either invented an income excess that does not exist or applied pre-ACA categorical gatekeeping that Louisiana abolished, and each therefore returned 0 where the ADULT expansion pathway returns eligible."
us,scenario_057,spouse_medicare_eligible,8,llm_error,age_disability,False,,"To be eligible for Medicare before age 65, a person must be entitled to Social Security Disability Insurance (SSDI) and have received it for 24 months, or have ESRD or ALS. Being disabled is not enough on its own. The spouse is 20 and has only $24 of generic `disability_benefits`. That is private or other disability income, not SSDI. Months of SSDI receipt and ESRD/ALS status are not listed, so they default to 0/false. The spouse fails both the age-65 test and the disability pathway. Every model read ""is disabled"" plus a small disability-benefit amount as satisfying the disability pathway."
us,scenario_057,spouse_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC does not cover women merely because they are young, disabled, or income-eligible. A woman must be pregnant, breastfeeding, or postpartum; the spouse has none of these statuses, and the household has no infants or children under age five."
-us,scenario_057,ssi,51,prompt_ambiguity,age_disability,False,,"The engine treats only the head (age 43, blind and disabled) as SSI aged, blind, or disabled. The 20-year-old spouse is not, so the head is an eligible individual with an ineligible spouse. The spouse's $24 a year of income is far below the deeming threshold, so the head gets the 2026 individual FBR of $994 a month, not the couple rate of about $1,491. The head's countable income is (866.67 − (20 − 1.17) − 65) / 2 = $391.42 a month, giving $602.58 a month, or $7,231 a year. Most models applied the couple FBR, which gives about $13,183; others wrongly found the household ineligible because they skipped the $65-plus-one-half earned income exclusion. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 7231.001465 to 13182.999023 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. An aged or blind member already receives SSI; the disabled 20-year-old would add a second federal benefit under the alternative reading. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)."
+us,scenario_057,ssi,52,prompt_ambiguity,age_disability,False,,"Under PolicyEngine, only the head (blind and disabled) counts as SSI aged/blind/disabled. The 20-year-old spouse is not SSI-ABD, so the head is an eligible individual with an ineligible spouse. That means the 2026 individual federal benefit rate of $994/month applies, not the roughly $1,491 eligible-couple rate. The spouse's $2/month of unearned income is below the deeming threshold, so none of it is deemed to the head. The head's own $1.17/month of unearned income uses up part of the $20 general exclusion, which leaves $18.83 for earnings. Countable income is ($866.67 − $18.83 − $65)/2 = $391.42, so the benefit is $994 − $391.42 = $602.58/month, or $7,231/year. Most models treated the household as an eligible couple and landed near $13,183. The rest returned $0 because they wrongly concluded that the income, the assets, or the absence of listed SSI receipt ruled the head out. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 7231.001465 to 13182.999023 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. An aged or blind member already receives SSI; the disabled 20-year-old would add a second federal benefit under the alternative reading. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)."
us,scenario_057,state_income_tax_before_refundable_credits,3,llm_error,state_local_rule,False,,"The trap is Louisiana's flat-tax reform, effective tax year 2025: Act 11 of the 2024 Third Extraordinary Session replaced the graduated 1.85/3.5/4.25% schedule and the $4,500-per-filer personal exemptions with a single 3% rate and a $25,000 standard deduction for married filing jointly (indexed from 2026), and Louisiana taxable income starts from federal AGI. This household's only taxable income is $10,400 of SSTB self-employment income, yielding federal AGI of about $9,665 after the deductible half of self-employment tax — far below the joint standard deduction — so Louisiana taxable income and tax are both zero. Both wrong answers subtracted $9,000, the repealed exemption figure, manufacturing a $665 taxable base that the current deduction fully absorbs."
us,scenario_057,state_refundable_credits,36,llm_error,state_local_rule,False,,"This is a married-filing-jointly couple with no children. The head is 43, which is inside the 25–64 age window for the childless federal EITC, and on a joint return only one spouse has to meet that age test. EITC earned income is $10,400 of SE income minus half of SE tax (about $735), so roughly $9,665. That is above the 2026 childless phase-in end point, so the federal EITC is the 2026 maximum of $664. Louisiana pays a refundable state EITC of 5% of the federal EITC, which gives $33.20. Most models missed it in one of three ways: they said Louisiana has no refundable EITC or that it needs children, they set the federal EITC to zero, or they used the wrong year's federal maximum or the wrong Louisiana match rate."
us,scenario_059,federal_income_tax_before_refundable_credits,13,llm_error,taxable_income_or_deductions,False,,"Provisional income is $25,600 + $18,552 = $44,152, only $152 above the $44,000 MFJ second-tier threshold. Taxable Social Security is therefore $6,000 + 0.85 × $152 = $6,129.20, which makes AGI $31,729.20. For 2026, OBBBA made the TCJA standard deduction permanent: $32,200 for MFJ plus $1,650 for each spouse aged 65+, for a total of $35,500. Each spouse also gets the new $6,000 senior deduction, adding $12,000. Total deductions of about $47,500 are far above AGI, so taxable income and tax are $0. Most models went wrong in one of two ways: they assumed the TCJA would sunset and used the pre-TCJA standard deduction plus personal exemptions (about $29–30.5k), or they overstated taxable Social Security by applying 85% to all benefits instead of using the tiered formula."
@@ -519,7 +519,7 @@ us,scenario_064,dependent2_chip_eligible,10,llm_error,health_coverage,False,,"De
us,scenario_064,dependent2_medicaid_eligible,6,prompt_ambiguity,household_unit_or_filing_status,False,,"Dependent 2 is 18, so the only pathway open is the older-child MAGI category. It is income-tested against the five-person tax household's MAGI, which the engine puts at 3.22× FPL (322%). That is above Wisconsin's Medicaid limit for ages 6–18, above the roughly 306% BadgerCare/CHIP ceiling the models quoted, and above that ceiling plus the 5-point disregard (about 311%). No other pathway applies: there is no disability, pregnancy, caretaker status or SSI. Every model that answered Yes assumed or computed household MAGI below about 300–306% FPL and treated the CHIP ceiling as the Medicaid child limit. Its income measure fell short of the engine's 322%. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 0.00; the alternative is 1.00. The prompt lists an 18-year-old 'Dependent' separately from the household's children without saying whose child they are. Read as a non-child tax dependent, the dependent's Medicaid household is their own under 42 CFR 435.603(f)(2)(i), with $0 of MAGI; the reference places them in the claiming filers' household. Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)."
us,scenario_064,dependent2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,dependent2_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_064,federal_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"The correct 2026 computation follows the OBBBA rules, under which the TCJA structure is permanent. AGI is $114,421.41: wages net of the 401(k) deferral, plus $19,350 of partnership income (the $14,000 SE partnership amount is part of it, not extra income), plus the $20,000 distribution, less the $16,800 farm loss, the $3,000 capital-loss limit and the $108 IRA deduction. From that, subtract the $32,200 standard deduction, a $510 QBI deduction (20% of $2,550 net business income) and the $13,899.29 overtime-premium deduction. That gives $67,812.12 of taxable income and $7,641.46 of tax, and $3,200 of CTC/ODC brings it to $4,441.46. Most models went wrong in one of these ways: they left out the overtime deduction, applied the expired TCJA-sunset rules, counted the $14,000 SE partnership income twice, subtracted the repealed $1,796 domestic production deduction, or added items that aren't part of this output, such as SE tax or the 10% early-withdrawal penalty."
+us,scenario_064,federal_income_tax_before_refundable_credits,48,llm_error,taxable_income_or_deductions,False,,"Getting the reference requires several steps. Exclude the $2,315 traditional 401(k) deferral from wages. Treat the $14,000 self-employment partnership income as part of the $19,350 partnership figure, not extra income. Net the $16,800 farm loss and a capital loss capped at $3,000 above the line, which gives AGI of $114,421. Then subtract the 2026 OBBBA $32,200 MFJ standard deduction, the $13,899 overtime-premium deduction and a $510 QBI deduction (20% of the $2,550 net business income). The repealed domestic production deduction is not allowed. On the $67,812 of taxable income, tax is $7,641, less a $2,200 CTC and two $500 other-dependent credits. Most wrong answers left out the overtime deduction, counted the $14,000 SE partnership income twice, used the repealed DPAD or pre-TCJA sunset law, or added the 10% early-withdrawal tax or SE tax, neither of which belongs in this measure."
us,scenario_064,federal_refundable_credits,6,llm_error,thresholds_rates,False,,"The refundable child credit is a residual, not an automatic payment: the $2,000-per-child CTC first offsets federal income tax, and only the portion left unused is refundable, subject to the $1,700-per-child ceiling and 15% of earned income above $2,500. This MFJ household's ~$98,810 of income (wages $97,295, $20,000 of 401(k) distributions, $19,350 partnership income, against a farm loss and a $3,000-capped capital loss) produces federal income tax before refundable credits far above $2,000, so the single under-17 child's entire CTC is absorbed nonrefundably, and the 18- and 27-year-old dependents generate only the $500 nonrefundable other-dependent credit. EITC is zero because income is multiples of the MFJ one-child phaseout ceiling, and no refundable AOTC (no students), recovery rebate, or refundable payroll tax credit applies, leaving $0. Every wrong model reported either the full CTC or the $1,700 ACTC ceiling as if refundability were automatic rather than conditional on unused credit."
us,scenario_064,free_school_meals_eligible,2,llm_error,taxable_income_or_deductions,False,,"The National School Lunch Program income test counts gross income by source — wages plus taxable retirement distributions — with no netting of farm losses, capital losses, or other tax-code offsets, and no deduction for premiums or medical expenses. PolicyEngine's countable income is $97,295 in wages plus $20,000 in taxable 401(k) distributions = $117,295 against an SPM-unit FPG of $38,680, a ratio of 3.03, far above the 130% free line ($50,284) and the 185% reduced-price line ($71,558). Wisconsin has no universal free meals and no SNAP/TANF categorical eligibility exists here, so the household lands in the PAID tier. The trap is importing tax-return netting (the -$16,800 farm loss and -$18,235 long-term capital loss) into a gross-income benefit test."
us,scenario_064,head_chip_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -527,7 +527,7 @@ us,scenario_064,head_medicaid_eligible,1,parse_contract_failure,missing_output,F
us,scenario_064,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,local_income_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_064,payroll_tax,18,llm_error,payroll_tax_base,False,,"The reference applies the employee FICA rates to the head's full listed gross wages: 6.2% × $97,295 = $6,032.29 and 1.45% × $97,295 = $1,410.78, for a total of $7,443.07. Wisconsin has no employee state payroll tax, wages are far below the Additional Medicare Tax threshold, and nobody else in the household has wages. Most models went wrong in one of two ways. Some made up reductions to the FICA base, either by treating the listed ESI or health premiums as Section 125 salary reductions or by subtracting traditional 401(k) deferrals, which are FICA wages under IRC 3121(v)(1)(A). Others reached $7,443.07 in their reasoning but then submitted a different number."
+us,scenario_064,payroll_tax,19,llm_error,payroll_tax_base,False,,"The reference applies 6.2% Social Security and 1.45% Medicare to the head's full listed gross wages of $97,295, which is below the 2026 wage cap. That gives $6,032.29 + $1,410.78 = $7,443.07. No other household member has wages, and Wisconsin has no employee state payroll tax. The main trap was shrinking the wage base by subtracting amounts the prompt never described as salary reductions. Five models subtracted the $19,648 employer-sponsored insurance premium as a pre-tax cafeteria-plan deduction, and others subtracted the traditional 401(k) deferral, which is FICA wages under IRC §3121(v)(1)(A), or the $1,560 health premium. Other models invented a Wisconsin employee payroll tax, padded the base with non-wage income, or gave a final number that didn't match the correct arithmetic they had shown."
us,scenario_064,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,self_employment_tax,11,llm_error,payroll_tax_base,False,,"The trap is the aggregation rule for net earnings from self-employment: section 1402(a) combines all of a person's trades and businesses, so the head's -$16,800 farm operations loss is netted against the $14,000 of partnership self-employment earnings, leaving non-positive net earnings and therefore no self-employment tax. A second trap is that the $19,350 partnership/S-corp line is not itself SE earnings — the $14,000 is the SE-earnings portion carved out of that distributive share, and S-corp and limited-partner shares are excluded under section 1402(a)(13) — so adding the two double-counts the same K-1 income. Every wrong model applied 92.35% x 15.3% to a positive base, either by dropping the farm loss entirely or by stacking the two partnership lines so the loss could not zero it out. Three models returned no value at all for this key."
us,scenario_064,snap,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -535,8 +535,8 @@ us,scenario_064,spouse_chip_eligible,2,parse_contract_failure,missing_output,Fal
us,scenario_064,spouse_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_064,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_064,ssi,20,prompt_ambiguity,age_disability,False,,"The answer turns on whether Dependent 1's ""is disabled"" flag meets SSI's aged/blind/disabled requirement. The engine sets is_ssi_aged_blind_disabled=False for all five members, Dependent 1 included, so the engine returns SSI of $0 regardless of income or resources. The prompt supports two readings. In one, ""is disabled"" is a general disability attribute, separate from SSA's SSI disability determination; this is the engine's reading. In the other, it means the person meets SSI's disability standard; eighteen models took this reading and paid Dependent 1 SSI, 31 answered $0 and two returned no value. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 11928.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading, fourteen of the eighteen models that paid SSI on the 51-model board of 2026-10-27 land within 10% of the $11,928 federal benefit (ten exactly); the other four applied an in-kind support reduction the facts do not support: one-third for deepseek-v4-pro, deepseek-v4-pro-0813 and inkling, and $350.33 a month for deepseek-v4.1-flash."
-us,scenario_064,state_income_tax_before_refundable_credits,50,llm_error,state_local_rule,False,"PolicyEngine carries the full $3,000 federally limited capital loss into Wisconsin AGI. Wisconsin Schedule WD and Publication 103 cap the net capital loss deduction at $500 for joint filers ($250 if married filing separately), so the engine is missing a $2,500 addition. With it, WI AGI would be $116,921.41, the sliding-scale deduction about $8,458.82, taxable income about $104,962.59 and tax about $4,758.33.","The engine sets Wisconsin AGI equal to federal AGI of $114,421.41: wages net of the $2,315 401(k) deferral, the $20,000 401(k) distribution, $19,350 partnership income, the -$16,800 farm loss, a -$3,000 capital loss and the -$108 IRA deduction. There is no SE tax deduction because the $14,000 SE partnership income nets negative against the farm loss. It subtracts the phased-down $8,953.27 sliding-scale deduction and five $700 exemptions to reach $101,968.14, then applies the 2026 joint schedule with the 4.4% bracket widened by 2025 law, with zero nonrefundable credits. Models missed mainly by using the old, narrower 4.4% bracket, inventing itemized, property-tax, married-couple or dependent credits, not phasing down the standard deduction, counting the SE partnership income twice, dropping exemptions, or adding back $2,500 of capital loss. Developer adjudication (2026-10-14): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). The judge flagged the reference on the hypothesis that Wisconsin limits the net capital loss deduction to $500, and labeled claude-opus-5.5's row reference_model_issue_fixed because it applied that limit. The $500 limit is Wis. Stat. 71.05(10)(c)1., which covers only taxable years beginning before January 1, 2023; for 2026, subd. 2. allows $3,000 on a joint return, the $3,000 the reference deducts, so Wisconsin adds nothing back and the reference's Wisconsin AGI of $114,421 stands. The rows are model errors. Reference affirmed (Wis. Stat. 71.05(10)(c)2. (for taxable years beginning after December 31, 2022, a net capital loss offsets ordinary income up to $3,000, or $1,500 for married persons filing separately); 2025 Wisconsin Schedule WD, line 28)."
+us,scenario_064,ssi,21,prompt_ambiguity,age_disability,False,,"Every substantive answer assumed that Dependent 1's generic ""is disabled"" flag meets SSI's disability criterion. In the engine, is_ssi_aged_blind_disabled=False for Dependent 1 and for every other member, so no one passes SSI's categorical aged, blind, or disabled test, and SSI is $0 no matter what the income or resources are. The prompt supports two readings. In one, ""is disabled"" means the person meets the SSA disability standard and qualifies categorically. In the other, it is a general disability flag, and the SSI-specific disability determination is an unlisted status that defaults to false. Dependent 1 also works 20 hours a week, which fits poorly with an SSI disability finding. The engine uses the second reading. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 11928.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading, fourteen of the eighteen models that paid SSI on the 51-model board of 2026-10-27 land within 10% of the $11,928 federal benefit (ten exactly); the other four applied an in-kind support reduction the facts do not support: one-third for deepseek-v4-pro, deepseek-v4-pro-0813 and inkling, and $350.33 a month for deepseek-v4.1-flash."
+us,scenario_064,state_income_tax_before_refundable_credits,51,llm_error,state_local_rule,False,"PolicyEngine carries the full federal $3,000 net capital loss into Wisconsin AGI. Wisconsin Schedule WD and DOR Publication 103 cap the net capital loss deduction against ordinary income at $500 ($250 married filing separately), so the reference omits a $2,500 capital-loss addition, which would add about $158 of tax (about $4,758).","The main difference is Wisconsin's 2026 rate schedule. 2025 Wisconsin Act 15 widened the 4.4% bracket, so joint filers pay 3.5% up to about $20.1k, 4.4% up to about $69.3k, and 5.3% only above that. That gives $4,599.62 on $101,968.14 of taxable income, versus about $4,860 under the old bracket ending near $40k. Reaching that taxable income requires: carrying federal AGI of $114,421.41 into Wisconsin (401(k) deferral and $108 IRA deducted; the farm loss wipes out the $14,000 SE partnership earnings, so there is no SE-tax deduction; no domestic-production, overtime, or health-premium adjustment); then subtracting the $8,953.27 sliding-scale standard deduction and five $700 exemptions; and allowing no nonrefundable credits. The wrong answers came from using the pre-Act-15 brackets, overstating the standard deduction or skipping its phase-down, counting the $14,000 SE partnership income on top of the $19,350, inventing credits, or treating exemptions as credits. Developer adjudication (2026-10-14): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). The judge flagged the reference on the hypothesis that Wisconsin limits the net capital loss deduction to $500, and labeled claude-opus-5.5's row reference_model_issue_fixed because it applied that limit. The $500 limit is Wis. Stat. 71.05(10)(c)1., which covers only taxable years beginning before January 1, 2023; for 2026, subd. 2. allows $3,000 on a joint return, the $3,000 the reference deducts, so Wisconsin adds nothing back and the reference's Wisconsin AGI of $114,421 stands. The rows are model errors. Reference affirmed (Wis. Stat. 71.05(10)(c)2. (for taxable years beginning after December 31, 2022, a net capital loss offsets ordinary income up to $3,000, or $1,500 for married persons filing separately); 2025 Wisconsin Schedule WD, line 28)."
us,scenario_064,state_refundable_credits,4,llm_error,state_local_rule,False,,"Wisconsin's refundable individual income tax credits are the Homestead Credit, the Wisconsin EITC, the Farmland Preservation Credit, and the Veterans and Surviving Spouses Property Tax Credit; this household qualifies for none, so wi_refundable_credits and therefore state_refundable_credits are $0. The WI EITC is a flat percentage of the federal EITC (4%/11%/34% by number of qualifying children), and the federal EITC is $0 at $97,295 of wages plus partnership and 401(k) distribution income, so the WI EITC is $0. The Homestead Credit is barred both by its household-income ceiling near $24,680 and by the absence of any rent or property tax paid (only a mortgage balance is listed), and the Farmland Preservation Credit requires a farmland preservation agreement or certified zoning, which is unlisted and therefore false. The trap is arriving at an exact $0 rather than assuming that a household with a farm loss and Wisconsin residency picks up some small residual refundable credit."
us,scenario_064,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_066,federal_refundable_credits,10,llm_error,credit_phaseout,False,,"The childless EITC has no minimum earned-income floor. It phases in at 7.65% from the first dollar of earnings, and a 40-year-old single filer falls inside the 25–64 age window. With $520 of wages and no investment income, the credit is 0.0765 × $520 = $39.78, and the other refundable credits are all zero. Nine models wrongly assumed the EITC needs a minimum or ""substantial"" earned income, or that it depends on tax liability, so they answered $0. One model got $39.78 and then replaced it with a made-up $234 from a table lookup."
@@ -545,13 +545,13 @@ us,scenario_066,snap,33,llm_error,thresholds_rates,False,,"This one-person Virgi
us,scenario_066,state_refundable_credits,38,llm_error,categorical_eligibility,False,,"The federal childless EITC phases in from the first dollar at 7.65%, so $520 of wages produces $39.78. Virginia's refundable EITC is still in effect for 2026, and its rate is 20% of the federal EITC, not the older 15%. That gives $39.78 × 0.20 = $7.96, paid whether or not the filer owes Virginia tax. Twenty-seven models answered $0: they called the Virginia EITC nonrefundable only, assumed it had expired, or zeroed out the federal EITC. The other eleven found the refundable credit but used the outdated 15% rate (about $5.97 to $6)."
us,scenario_067,dependent1_medicaid_eligible,22,prompt_ambiguity,household_unit_or_filing_status,False,"PolicyEngine sets dependent1's Medicaid MAGI income level from the dependent's own income, which is 0.00x FPL, instead of using the household of the parents who claim them on a joint return. Under 42 CFR 435.603(f)(2) and (d), that household's income includes the parents' roughly $87,302 MAGI, about 3x FPL for a household of 3, so the dependent would be above Indiana's 138% FPL expansion limit.","The answers split on whose income sets dependent1's Medicaid MAGI level. All 22 models counted the parents' joint-return MAGI of about $87,302 (wages plus pension) toward the 23-year-old tax dependent, which put them at about 3x FPL and ruled out expansion coverage. PolicyEngine puts dependent1's MAGI income level at 0.00x FPL because it does not count the $10,800 disability benefits or the parents' income. That places dependent1 in Indiana's ACA expansion adult category, which has a 138% FPL limit and no resource test. A few models made separate mistakes: some said $10,800 alone was over the single-adult limit, and some looked only at SSI-linked or asset-tested pathways. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 1.00; the alternative is 0.00. The prompt lists an adult 'Dependent' without saying whether that person is the claiming filers' child. If the dependent is their child, 42 CFR 435.603(f)(2) puts the dependent in the parents' Medicaid household, whose combined MAGI decides eligibility; the reference treats the dependent as a non-child whose household is their own (42 CFR 435.603(f)(2)(i), (f)(3)). Reference depends on an unlisted input; output excluded from scoring (the relationship of an adult tax dependent to the filers who claim them)."
us,scenario_067,dependent1_medicare_eligible,13,llm_error,age_disability,False,,"Medicare eligibility under age 65 does not come from being disabled. It requires 24 months of entitlement to Social Security Disability Insurance (or ESRD/ALS). PolicyEngine checks this with age ≥ 65, or positive SSDI with months receiving SSDI ≥ 24. It does not model the ESRD or ALS routes. Dependent 1's $10,800 is a generic 'disability benefits' input, not SSDI. SSDI and months of SSDI receipt are unlisted, so they default to 0. At age 23, Dependent 1 fails every pathway, yet every model treated 'is disabled + disability benefits' as enough for Medicare."
-us,scenario_067,federal_income_tax_before_refundable_credits,42,llm_error,thresholds_rates,False,,"The reference uses current law, under which OBBBA made the TCJA structure permanent. AGI is $87,302 (wages plus pension). The dependent's own $10,800 of disability benefits is not on the parents' return. The 2026 MFJ standard deduction of $32,200 is larger than itemizing, because no mortgage interest is listed and net medical is only about $16,621. That leaves taxable income of $55,102, taxed at 10% up to $24,800 and 12% above it, for $6,116.24. The household then gets a $500 nonrefundable Credit for Other Dependents for the permanently disabled 23-year-old qualifying child, giving $5,616.24. The models went wrong in three main ways: using stale standard deduction or bracket figures (2024/2025 values, or a TCJA-sunset regime with personal exemptions and a 15% bracket); deducting the listed auto loan interest, which no fact makes qualified vehicle loan interest; and either dropping the $500 ODC or adding the dependent's benefits to the parents' income."
+us,scenario_067,federal_income_tax_before_refundable_credits,43,llm_error,thresholds_rates,False,,"The reference uses post-OBBBA 2026 law for married filing jointly. AGI is $87,302 (wages plus pension; the dependent's disability benefits stay out of the filers' income). The $32,200 standard deduction leaves $55,102 of taxable income. Tax is 10% on $24,800 plus 12% on $30,302, or $6,116.24. Subtracting the $500 credit for other dependents for the permanently disabled 23-year-old gives $5,616.24. Models missed in four main ways: applying the expired pre-TCJA structure (personal exemptions, 15% bracket), using stale or guessed standard-deduction and bracket figures, deducting the $1,845 of auto loan interest without facts showing a qualifying vehicle, or adding the dependent's benefits to AGI. Several also dropped the $500 credit or swapped in the $2,000 CTC."
us,scenario_067,federal_refundable_credits,2,llm_error,categorical_eligibility,False,,"The 23-year-old disabled household member does not generate a refundable Child Tax Credit because the CTC requires a qualifying child under age 17. With no CTC-qualifying child and no eligibility for the other refundable credit components, federal refundable credits equal $0."
us,scenario_067,head_medicare_eligible,1,llm_error,age_disability,False,,"Ordinary age-based Medicare eligibility begins at age 65, and the 60-year-old head is below that threshold. No qualifying disability-based Medicare pathway or other exception is supplied, so the head is not Medicare eligible."
-us,scenario_067,local_income_tax,51,prompt_ambiguity,state_local_rule,False,,"PolicyEngine's local_income_tax includes Indiana county income tax. With no county given, it uses Adams County, which levies 1.6% on Indiana AGI: $87,302 federal AGI minus $3,000 in exemptions ($1,000 per person × 3) = $84,302, and $84,302 × 1.6% = $1,348.83. The prompt defines the output by listing only NYC income tax, Philadelphia wage tax, Kansas City earnings tax and St. Louis earnings tax, and it names no county. One reading treats that list as complete, so an Indiana household owes $0; the other treats it as examples (""where applicable""), so the Indiana county tax counts. Almost every model took the first reading and answered $0. A few instead claimed Indiana has no local income tax, which is false, and one included the county tax and answered $1,800. Developer adjudication (2026-10-10): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state. Reference depends on an unlisted input; output excluded from scoring (county of residence (Indiana county income tax))."
+us,scenario_067,local_income_tax,52,prompt_ambiguity,state_local_rule,False,,"The prompt defines local_income_tax by naming four city taxes: NYC, Philadelphia, Kansas City and St. Louis. Under one reading that list is complete, so an Indiana household owes $0. Under the other reading the list is only examples ('where applicable'), and PolicyEngine's local_income_tax also includes Indiana county income tax. The prompt gave no county, so PolicyEngine used its default, Adams County, at 1.6%. It applied that rate to Indiana AGI of $84,302 ($87,302 federal AGI minus three $1,000 personal exemptions), giving $1,348.83. Almost every model took the first reading. A few wrongly said Indiana has no local income tax at all, and one model included the county tax but guessed the amount. Developer adjudication (2026-10-10): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The prompt places the household in Indiana but states no county. Every Indiana county levies its own local income tax (IC 6-3.6), at rates that differ by county. policyengine-us 2.15.17 left county income tax out of local_income_tax, so the frozen reference was $0.00. From policyengine-us 2.23.1 (PolicyEngine/policyengine-us#9694, which standardized local income tax) the engine counts Indiana county income tax in local_income_tax, at the rate of the county it assigns a household with no stated county. The reference therefore depends on the household's county, which the prompt does not state. Reference depends on an unlisted input; output excluded from scoring (county of residence (Indiana county income tax))."
us,scenario_067,payroll_tax,6,llm_error,payroll_tax_base,False,,"The head is the only person with wages. Their $82,360 is taxed at 6.2% for Social Security ($5,106.32) and 1.45% for Medicare ($1,194.22), for $6,300.54 total. Those wages are below the 2026 Social Security wage base and the $250,000 Additional Medicare Tax threshold, and Indiana has no employee state payroll tax. The spouse has no listed wages, so the spouse's wages are $0, and neither the pension nor the disability benefits are FICA wages. The models went wrong by adding wages for the spouse, putting non-wage income into the tax base, or making rounding and arithmetic errors in the simple 7.65% calculation."
us,scenario_067,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_067,ssi,19,prompt_ambiguity,age_disability,False,"The prompt lists Dependent 1 as ""is disabled,"" but the engine's SSI disability input never reached is_ssi_aged_blind_disabled, which is False for dependent1. Under 42 U.S.C. §1382c(a)(1)(A), a disabled adult whose countable unearned income is $880/month, below the 2026 FBR of $994, gets $114/month, or $1,368/year.","The case turns on whether anyone meets SSI's categorical test. In the engine, is_ssi_aged_blind_disabled=False for all three members, including the 23-year-old shown in the prompt as ""is disabled,"" so SSI is $0 no matter what the income is. All 19 models treated that disabled flag as meeting SSI's disability criterion, applied no parental deeming (Dependent 1 is an adult), and netted disability income against the federal benefit rate (FBR), with sixteen applying the $20 general exclusion monthly, glm-5.2 and grok-build-0.1 applying it once annually, and minimax-m3 omitting it. Thirteen models landed on exactly (994 − 880) × 12 = $1,368. The rest added their own FBR or exclusion errors. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned reference_data_issue_fixed; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1368.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading 14 of the 17 models on the 46-model board that paid SSI land within 10% of $1,368 (11 exactly; nine of twelve, six exactly, when the output was excluded on 2026-09-05); glm-5.2, grok-build-0.1 and minimax-m3 used wrong rates or annualised the monthly exclusion. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)."
+us,scenario_067,ssi,20,prompt_ambiguity,age_disability,False,"The question lists Dependent 1 as 'is disabled' with zero earned income, which is well below SGA under 20 CFR 416.974 and meets 42 U.S.C. 1382c(a)(3), but the engine returned is_ssi_aged_blind_disabled=False and its derivation says nobody is flagged disabled. This suggests the benchmark's is_disabled input is not reaching PolicyEngine's is_ssi_disabled variable, or that variable applies an extra condition the input does not satisfy.","All 20 models treated Dependent 1's 'is disabled' flag as meeting SSI's disability test, then computed the federal benefit rate minus countable unearned income. A correct 2026 calculation gives $994 − ($900 − $20) = $114 a month, or $1,368 a year. The engine sets is_ssi_aged_blind_disabled=False for all three members: Head (60) and Spouse (58) are under 65, and Dependent 1 is not treated as SSI-disabled. No one passes the aged/blind/disabled test, so SSI is $0 before any income or resource test. The models differ from the reference only on this disability determination, plus FBR or exclusion errors in a few answers. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1368.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. Under the alternative reading 14 of the 17 models on the 46-model board that paid SSI land within 10% of $1,368 (11 exactly; nine of twelve, six exactly, when the output was excluded on 2026-09-05); glm-5.2, grok-build-0.1 and minimax-m3 used wrong rates or annualised the monthly exclusion. Reference depends on an unlisted input; output excluded from scoring (42 U.S.C. 1382c(a)(3)(A); 20 CFR 416.905)."
us,scenario_067,state_income_tax_before_refundable_credits,40,llm_error,thresholds_rates,False,,"Indiana tax here is a two-step calculation. Start from federal AGI of $87,302 (wages plus taxable pension; the dependent's disability benefits are not on the joint return) and subtract three $1,000 personal exemptions, giving $84,302. Then apply the 2026 flat rate of 2.95%, which gives $2,486.91. The 23-year-old is not under 19 and is not a full-time student, so the additional $1,500 dependent-child exemption does not apply, and being disabled does not change that. Indiana has no standard deduction, no itemized deductions, and no general private-pension subtraction. Most models went wrong in one of two ways: they used the wrong rate (2.9%, 3.0%, 3.05%) or added an extra $1,500–$2,000 of exemptions."
us,scenario_067,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_068,federal_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"The reference uses wages of $34,083.53 as AGI. The $73,000 of educational assistance is excluded entirely, and the ESI premiums are not subtracted from the listed gross wages. It then subtracts the 2026 OBBBA single standard deduction of $16,100, leaving taxable income of $17,983.53. Tax is 10% of the first $12,400 plus 12% of the remaining $5,583.53, which gives $1,910.02, with no nonrefundable credits. The wrong answers fall into three groups. Some models put part or all of the educational assistance into income. Some used TCJA-sunset rules or 2024/2025 standard deductions and brackets instead of the permanent 2026 amounts. Others subtracted the refundable EITC, or credits the filer doesn't have, from a tax line measured before refundable credits."
@@ -568,7 +568,7 @@ us,scenario_070,federal_income_tax_before_refundable_credits,40,llm_error,credit
us,scenario_070,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"The refundable Child Tax Credit requires at least one qualifying child; this household contains only a single 57-year-old adult. With no qualifying child and income above the childless EITC phaseout range, every refundable federal credit component is zero."
us,scenario_070,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"Illinois’s ACA adult Medicaid pathway requires income no greater than 138% FPL, while the head’s MAGI income is 2.27 times FPL. The head also qualifies through no non-MAGI pathway, so the Medicaid category is NONE and eligibility is No."
us,scenario_070,payroll_tax,5,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes apply to the full $38,000 of wages because the prompt lists $300 of health insurance premiums but never identifies them as a pre-tax cafeteria-plan deduction from FICA wages. The correct computation is $38,000 × 6.2% = $2,356 plus $38,000 × 1.45% = $551, totaling $2,907; the Additional Medicare Tax and Illinois employee payroll taxes are zero."
-us,scenario_070,state_income_tax_before_refundable_credits,38,llm_error,thresholds_rates,False,,"The Illinois calculation is simple. Base income is federal AGI of about $36,270 (wages minus the traditional 401(k), plus interest, minus the traditional IRA and student loan interest deductions). Subtract one inflation-indexed 2026 personal exemption of $2,925 and apply the flat 4.95% rate. There are no nonrefundable credits for this renter. Most models got the base right but used an older exemption amount: $2,850 (2025), $2,775 (2024) or $2,425 (2023). Others made structural errors: inventing a standard deduction, treating the exemption as a credit, adding back above-the-line deductions, excluding the health premium from AGI, or claiming a renter's credit."
+us,scenario_070,state_income_tax_before_refundable_credits,39,llm_error,thresholds_rates,False,,"Illinois tax here is 4.95% × (federal AGI $36,269.67 − the 2026 inflation-indexed personal exemption of $2,925) = $1,650.56. Illinois has no standard deduction, no add-back for IRA or student-loan deductions, and no nonrefundable credit available to a renter in this case. Most models got AGI right but used an outdated exemption: $2,850 (2025), $2,775 (2024) or $2,425 (2023). That moves the answer by $4 to $25. The rest made bigger errors: they dropped the student loan interest deduction, invented standard deductions or renter credits, applied the exemption as a credit, or left the exemption out entirely."
us,scenario_070,state_refundable_credits,1,llm_error,credit_phaseout,False,,"Illinois's Earned Income Credit is calculated as a percentage of the federal EITC, so expanding the eligible age range does not create a credit when the federal EITC amount is zero. At this childless filer's income, the federal EITC has fully phased out, making the Illinois refundable credit zero."
us,scenario_071,head_medicaid_eligible,8,llm_error,categorical_eligibility,False,,"Age 76 does not itself establish Medicaid eligibility, and this person qualifies through none of the Medicaid pathways represented by the engine. Their MAGI income is 2.28 times FPL, they receive no SSI, and the engine assigns medicaid_category = NONE; Medicare eligibility, housing assistance, and the mere availability of aged or spend-down programs do not create Medicaid eligibility."
us,scenario_071,payroll_tax,1,llm_error,payroll_tax_base,False,,"Payroll tax applies to listed wages and other covered earnings, not to work hours by themselves. The household has no listed wage or self-employment income, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll tax bases are all zero, yielding payroll_tax of $0."
@@ -606,19 +606,19 @@ us,scenario_077,head_wic_eligible,1,parse_contract_failure,missing_output,False,
us,scenario_077,payroll_tax,9,llm_error,payroll_tax_base,False,,"Employee Social Security and Medicare taxes are computed from the applicable payroll-tax wage base, and the reference calculation applies the 6.2% and 1.45% employee rates to the full $26,006 of wages. The employer-sponsored insurance premium input does not reduce that wage base in this calculation, producing $1,612.38 of Social Security tax plus $377.09 of Medicare tax, or $1,989.47."
us,scenario_077,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_077,snap,5,llm_error,categorical_eligibility,False,,"Louisiana’s broad-based categorical eligibility gross-income screen does not itself guarantee a SNAP allotment; the household must still produce an eligible benefit under the net-income and allotment calculations. SNAP countable income does not subtract capital losses, a mortgage balance is not a shelter expense, and a non-elderly, nondisabled household cannot deduct medical expenses. With no listed deductible shelter payment and countable wages of $26,006, this one-person household receives $0."
-us,scenario_077,state_income_tax_before_refundable_credits,51,reference_later_law,taxable_income_or_deductions,False,,"Louisiana's 2025 reform set a flat 3% rate and a $12,500 single standard deduction, and repealed the personal exemption. Louisiana ended the federal income tax deduction after 2021. Starting in 2026 the standard deduction is indexed for inflation under R.S. 47:294(B). The frozen reference, computed on policyengine-us 2.15.17, uses $12,835: ($23,006.18 AGI − $12,835) × 3% = $305.14. Louisiana first published its 2026 return deduction, $12,838, in RIB 26-019; that gives $305.05, and this output is excluded from scoring under d994. Of the 51 models, 23 answered $315.18 or $315, consistent with the unindexed $12,500 deduction. The others applied pre-reform law (the $4,500 exemption, the 1.85%/3.5% brackets, the federal tax deduction), subtracted ESI premiums or the federal standard deduction, ignored the $3,000 capital-loss cap, or wrongly said Louisiana has no income tax. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_later_law (taxable_income_or_deductions). The frozen reference is 305.14; the value under the published figure is 305.05. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana). Reference depends on law published after the reference freeze; output excluded from scoring (Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published)."
+us,scenario_077,state_income_tax_before_refundable_credits,52,reference_later_law,taxable_income_or_deductions,False,,"Louisiana's 2025 reform set a flat 3% rate on federal AGI less a single standard deduction, and that deduction is inflation-indexed to $12,835 for 2026. AGI is $23,006.18: $26,006 of wages less the $3,000 capital-loss limit, with no subtraction for employer insurance premiums. So taxable income is $10,171.18 and tax is $305.14. Most models used the un-indexed 2025 amount of $12,500 and got $315.18. The rest either used repealed pre-reform rules (the 1.85%/3.5% brackets, the $4,500 exemption, the deduction for federal tax paid), cut AGI by the employer insurance premiums, or claimed Louisiana has no income tax. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_later_law (taxable_income_or_deductions). The frozen reference is 305.14; the value under the published figure is 305.05. The reference rests on a 2026 Louisiana standard deduction of $12,835, which policyengine-us computed from published CPI-U under R.S. 47:294(B) but which no Louisiana publication stated before the 2026-07-03 freeze; Louisiana's first 2026 return amount, $12,838, came in Revenue Information Bulletin 26-019 on 2026-09-28. Excluded under decision d994, which keeps the published-amounts convention for every other reference (reference_audit/2026-10-05-louisiana). Reference depends on law published after the reference freeze; output excluded from scoring (Louisiana Department of Revenue, Revenue Information Bulletin 26-019, dated 2026-09-28: Louisiana's 2026 standard deduction is $12,838 for single and separate filers and $25,676 for the other statuses, the first 2026 return amount Louisiana published)."
us,scenario_077,state_refundable_credits,4,llm_error,state_local_rule,False,,"The models applied Louisiana’s former refundable earned income tax credit, equal to 5% of the federal EITC, to tax year 2026. Louisiana no longer provides that refundable credit in 2026, so the Louisiana refundable-credits computation returns zero regardless of an estimated federal EITC amount."
us,scenario_077,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_078,federal_income_tax_before_refundable_credits,51,prompt_ambiguity,taxable_income_or_deductions,False,,"Under OBBBA, TCJA's rates stay in place for 2026, and the SALT cap is $40,400 for MAGI below $505,000. The frozen reference combines $26,460 in real estate tax with the engine's Maryland withholding estimate, which includes state and county income tax. That total is above the cap, so the SALT deduction is the full $40,400. Itemized deductions are therefore $68,610 ($40,400 SALT plus $28,210 mortgage interest), taxable income is $131,526.91, and the 2026 single brackets (10/12/22/24% with breakpoints at $12,400, $50,400 and $105,700) give $24,164.46. The models fell into four groups: counting only property tax (about $27,510), adding state income tax but not the county tax (about $25,845), using the old $10,000 SALT cap (about $31,400), or assuming TCJA expired (pre-TCJA brackets plus a personal exemption). Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,164.46; the alternative is 23,772.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
+us,scenario_078,federal_income_tax_before_refundable_credits,52,prompt_ambiguity,taxable_income_or_deductions,False,,"The trap is the size of the SALT itemized deduction. Under post-OBBBA 2026 law the SALT cap is $40,400, and MAGI of about $200k is far below the $505,000 phasedown. The deductible amount is $26,460 of real estate tax plus Maryland state income tax plus Maryland county income tax, and that total exceeds the cap, so SALT is $40,400. Adding $28,210 of mortgage interest gives itemized deductions of $68,610 and taxable income of $131,526.91. The 2026 single brackets (10% to $12,400, 12% to $50,400, 22% to $105,700, then 24%) produce $24,164.46. Most models left state income tax out of SALT, counted only Maryland state tax without the county piggyback tax, used the old $10,000 cap, assumed TCJA had expired, or took the standard deduction. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,164.46; the alternative is 23,772.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
us,scenario_078,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_078,local_income_tax,1,llm_error,state_local_rule,False,,"The requested local_income_tax output is limited to the separately modeled local taxes listed in the prompt: NYC, Philadelphia, Kansas City, and St. Louis. Maryland county income tax is not included in this output, so a Maryland household outside those modeled jurisdictions has a value of zero."
us,scenario_078,payroll_tax,27,llm_error,thresholds_rates,False,,"The correct payroll tax uses the 2026 Social Security wage base of $184,500: 6.2% × $184,500 = $11,439, plus 1.45% Medicare on all $198,505 of wages ($2,878.32), for $14,317.32. Wages are below the $200,000 Additional Medicare Tax threshold, and Maryland charges no mandatory employee payroll tax in 2026. Most wrong answers used an outdated or wrongly projected wage base. Others added a Maryland FAMLI contribution, which is not collected in 2026, or invented a Maryland payroll tax. Several models worked out exactly $14,317.32 and then submitted a different number."
-us,scenario_078,state_income_tax_before_refundable_credits,38,llm_error,taxable_income_or_deductions,False,,"Maryland AGI subtracts the $1,632 state tax refund, which brings it to $198,505. Maryland itemized deductions are the $28,210 of mortgage interest plus the full $26,460 of property tax, with no federal $10,000 SALT cap. That $54,670 is reduced by 7.5% of federal AGI above $200,000 ($10.27), leaving $54,659.73. The personal exemption is fully phased out above $150,000, so it is $0. That gives taxable income of $143,845.27. The state brackets (2/3/4/4.75/5/5.25%) on that amount give $6,936.88, and county income tax is not included. Most wrong answers took the small standard deduction instead of itemizing, applied the federal SALT cap to the Maryland deduction, kept the $1,632 refund in Maryland AGI, claimed an exemption that had already phased out, or added county tax."
+us,scenario_078,state_income_tax_before_refundable_credits,39,llm_error,taxable_income_or_deductions,False,,"The reference uses Maryland AGI of $198,505, which is federal AGI of $200,137 minus the $1,632 state tax refund. From that it subtracts Maryland itemized deductions: $28,210 mortgage interest plus the full $26,460 of property taxes, with no $10,000 SALT cap. That $54,670 is reduced by 7.5% of federal AGI above $200,000, which is only $10.27. The single filer's personal exemption is $0 because income is above $150,000. Taxable income of $143,845.27, run through the 2/3/4/4.75/5/5.25% brackets, gives $6,936.88. Wrong answers clustered around five mistakes: using the small standard deduction instead of itemizing; capping property taxes at $10,000; skipping the refund subtraction; claiming a personal exemption, using the wrong phase-out, or making bracket arithmetic errors; and adding county income tax."
us,scenario_078,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_079,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The operative rule is categorical, not income-based: the head's blindness, disability, and $7,560 of SSDI leave countable income below the SSI federal benefit rate, producing $5,286 of SSI, and Arizona is a 1634 state where SSI receipt automatically confers Medicaid with no MAGI test. Both models ran an income/coverage screen instead of the SSI-recipient category, and neither recognized that SSI here is a computed output of the listed facts rather than an unlisted input. The screen they ran also points to eligibility rather than away from it — 0.35 x FPL is far under Arizona's 138% FPL expansion threshold — so 'No' is unreachable by either the categorical or the MAGI route."
us,scenario_079,head_medicare_eligible,27,prompt_ambiguity,age_disability,False,,"The head is 58, below the Medicare age threshold of 65. Someone under 65 qualifies through disability only after 24 months of SSDI entitlement (or with ESRD/ALS). The household facts list a $7,560 SSDI amount but no months of SSDI receipt and no ESRD/ALS status. Under the instruction to treat unlisted numeric inputs as 0 and unlisted statuses as false, the waiting-period count is 0, so the disability pathway fails. Every model treated SSDI receipt, or a disability flag, as enough on its own, or assumed the 24-month waiting period had already been met, and answered Yes where the frozen reference is No. The reference depends on the unlisted months-of-receipt input, so this output is not scored for any model. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it."
us,scenario_079,local_income_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_079,snap,51,llm_error,state_local_rule,False,,"Two steps separate the reference from the wrong answers. First, the household's SSI of $881/month is countable unearned income for SNAP (2026 couple FBR $1,491 minus $610 of countable SSDI), so gross income is $1,511/month, not the $630 of SSDI alone. Second, Arizona gives elderly/disabled households with more than $35/month in medical costs a $145 standard medical deduction in place of the $1 actual excess ($36 of other medical expenses − $35), since the engine does not count over-the-counter health expenses. With the $209 standard deduction, net income is $1,157, the 30% contribution is $348, and the benefit is $546 − $348 = $198/month, or $2,376 a year. Most models either left out SSI, which overstates the benefit (~$5,100), or counted SSI but used the $26 actual medical excess, which understates it (~$1,956)."
+us,scenario_079,snap,52,llm_error,state_local_rule,False,,"Two steps separate the $198/month reference from the wrong answers. First, the couple receives $881/month of SSI: the $1,491 couple benefit rate minus ($630 SSDI − $20 exclusion). That SSI counts as SNAP unearned income alongside the SSDI, for $1,511 gross. Second, Arizona gives elderly/disabled households with more than $35 of monthly medical costs a $145 standard medical deduction, which replaces the $26 actual excess ($61 − $35). With the $209 standard deduction, net income is $1,157. The contribution is 30% rounded up to $348, and $546 − $348 = $198/month, or $2,376/year. Every model that included SSI still used the $26 actual medical excess and landed near $1,960. Most others left out SSI and landed near $5,000. The $0 answers wrongly applied asset, income, shelter-cost, or SSI cash-out barriers."
us,scenario_079,spouse_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The spouse is blind and disabled with zero countable income and $1,045 in resources, so she receives SSI ($5,286 in the household trace); Arizona is a 1634 state, where SSI receipt confers automatic Medicaid through the SSI-recipient category with no separate income test. Both models evaluated only an income/MAGI-style adult pathway and never checked the SSI-recipient categorical route that the engine actually used (medicaid_category = SSI_RECIPIENT). The error is compounded because even the MAGI route qualifies here: household MAGI is 0.35 × FPL, far under Arizona's 138% FPL expansion-adult ceiling, and Indian Health Service coverage is not minimum essential coverage that bars Medicaid. Zero-income, SSI-receiving disabled adults in an expansion state are Medicaid-eligible on two independent pathways, so ""No"" is unreachable from the stated facts."
us,scenario_079,spouse_medicare_eligible,4,llm_error,age_disability,False,,"The spouse is 55, so the only route to Medicare is the disability pathway. That route needs 24 months of Social Security disability (SSDI) entitlement, or ESRD/ALS. Being disabled or blind does not qualify anyone for Medicare by itself. The spouse has no Social Security disability income at all (the $7,560 belongs to the head), and no months of SSDI receipt or ESRD are listed, so all of these inputs default to zero or false. Every model treated ""is disabled"" as enough for Medicare and skipped the SSDI entitlement and 24-month waiting-period requirement."
us,scenario_079,ssi,35,llm_error,thresholds_rates,False,,"Both spouses are blind and disabled, so they are an eligible SSI couple. Their only income is the head's $630/month SSDI, which becomes $610/month countable after the $20 general exclusion. Their $1,045 bank balance is under the $3,000 couple resource limit, and the vehicle and home are excluded. The 2026 couple federal benefit rate is $1,491/month, so the benefit is $881/month ($440.50 per spouse) and $10,572 a year. Nearly every wrong answer comes from using a stale or mis-projected couple rate ($1,415 from 2024, $1,450 from 2025, or $1,482–$1,490). The remaining wrong answers come from wrongly finding the couple over the income or resource limit, submitting a value that differs from the model's own reasoning, or giving no output."
@@ -639,7 +639,7 @@ us,scenario_080,ssi,2,llm_error,categorical_eligibility,False,,"SSI requires the
us,scenario_080,state_income_tax_before_refundable_credits,8,llm_error,credit_phaseout,False,,"Pennsylvania first applies its 3.07% rate to the $940 of taxable dividends and gains, then applies the nonrefundable Tax Forgiveness credit in computing income tax before refundable credits. The household's eligibility income remains within the range for 100% forgiveness, so the credit eliminates the preliminary tax and yields $0."
us,scenario_080,state_refundable_credits,2,llm_error,state_local_rule,False,,"Pennsylvania Schedule SP tax forgiveness is a nonrefundable credit that reduces Pennsylvania income-tax liability; it is not included in state refundable credits. With no qualifying Pennsylvania refundable credit, the requested aggregate is $0."
us,scenario_080,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_081,federal_income_tax_before_refundable_credits,51,prompt_ambiguity,taxable_income_or_deductions,False,,"The reference itemizes $31,027.41. That is $16,900 of mortgage interest plus $14,127.41 of SALT: $5,643 of property tax and $8,484.41 of estimated Massachusetts income tax withheld (5% of AGI above the $4,400 exemption). The SALT total is far below the 2026 OBBBA cap of $40,400. The $6 of REIT income enters only the $1.24 QBI deduction and is not part of AGI. Taxable income of $143,059.47 on the 2026 single brackets (10% to $12,400, 12% to $50,400, 22% to $105,700, then 24%) gives $26,932.27. Some answers omit state income tax from SALT, apply the expired $10,000 SALT cap, assume the TCJA sunset, or subtract employer insurance premiums from the listed wages; others use a computed Massachusetts liability in SALT. The output is excluded from scoring because state income tax withheld or paid is unspecified. The record gives alternative readings of $26,991.31 with the historical liability, $26,992.64 with the corrected liability, and $28,763.81 with no state income tax paid. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 26,932.27; the alternative is 26,991.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Massachusetts withholding on federal AGI ($8,484.41), while read as the household paying its 2026 Massachusetts income tax during the year the deduction takes the engine's liability ($8,238.41). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)."
+us,scenario_081,federal_income_tax_before_refundable_credits,52,prompt_ambiguity,taxable_income_or_deductions,False,,"This filer itemizes. The deduction is $16,900 of mortgage interest plus SALT of $14,127.41, well under the 2026 OBBBA SALT cap of $40,400. The SALT figure is $5,643 of real estate tax plus PolicyEngine's estimated Massachusetts withholding: 5% × (federal AGI $174,088.12 − $4,400) = $8,484.41. After the $1.24 QBI deduction, taxable income is $143,059.47, taxed on the 2026 brackets (10%/12%/22%/24% with breaks at $12,400, $50,400 and $105,700) to give $26,932.27. The wrong answers fall into four groups: models that left state income tax out of SALT, models that used the old $10,000 SALT cap, models that applied pre-TCJA law (personal exemptions and 15/25/28% brackets), and near-miss models that used Massachusetts tax liability (about $8,238) or some other figure instead of the $8,484.41 withholding estimate. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 26,932.27; the alternative is 26,991.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Massachusetts withholding on federal AGI ($8,484.41), while read as the household paying its 2026 Massachusetts income tax during the year the deduction takes the engine's liability ($8,238.41). Found in the 2026-10-05 audit of the state income tax in SALT (reference_audit/2026-10-05, scripts/sweep_salt_withholding.py). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set)."
us,scenario_081,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_081,payroll_tax,41,prompt_ambiguity,state_local_rule,False,,"The reference has two parts. First, employee FICA on the full $175,002 of wages: $10,850.12 Social Security plus $2,537.53 Medicare, with no Additional Medicare Tax because wages are under $200,000. Second, the Massachusetts PFML employee contribution of $805.01: 0.46% (the full 0.18% family rate plus 40% of the 0.70% medical rate) × $175,002. The prompt's phrase 'mandatory employee state payroll taxes' can be read two ways. Under one reading the PFML employee share counts, which is what the engine does by assuming the employer withholds the maximum. Under the other it doesn't, because Massachusetts lets employers deduct less of it or none, so the FICA-only $13,387.65 cluster follows from that reading. The remaining models included PFML at a stale or misallocated rate, cut FICA wages by the ESI premium, used a wrong Medicare rate, or submitted numbers that don't match their own arithmetic. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 14,192.66; the alternative is 13,387.65. The prompt counts mandatory employee state payroll taxes; Massachusetts PFML contributions are the employer's, which may deduct the employee share from wages but need not (M.G.L. c. 175M, s. 6(a) and (c); 458 CMR 2.05(5)), and policyengine-us 2.15.17 counts the largest share Massachusetts's published 2026 rates let the employer deduct ($805.01); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
us,scenario_081,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -650,14 +650,14 @@ us,scenario_082,child1_early_head_start_eligible,6,llm_error,categorical_eligibi
us,scenario_082,child1_medicaid_eligible,6,llm_error,health_coverage,False,,"The trap is that New York's generous children's coverage above 154% FPL runs through Child Health Plus — the state's separate CHIP program — not through Medicaid: NY Medicaid covers a child age 1–5 only up to 154% FPL, and the 400–405% FPL figures the models cited are the CHIP ceiling. The second half of the trap is income: Medicaid MAGI here is about $117.8k (543% FPL for a two-person household) because the $33,000 long-term capital loss is limited to a $3,000 deduction against ordinary income and because $20,580 of qualified and non-qualified dividends, taxable interest, short-term gain, and the state tax refund all count, while the $21,208 ESI premium is not subtracted from MAGI. Every model either imported a CHIP-scale threshold into Medicaid, understated MAGI to roughly $90k–$99k, or asserted child eligibility with no income test at all. At 543% FPL the child clears no Medicaid category (medicaid_category = NONE), so is_medicaid_eligible is False."
us,scenario_082,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_082,child1_wic_eligible,2,llm_error,thresholds_rates,False,,"WIC requires both categorical eligibility and financial eligibility; being age 1 satisfies only the categorical test. The household's approximately $90,132 annual income exceeds the 2026 WIC limit of 185% of the federal poverty guideline for a two-person household, so the child is not eligible."
-us,scenario_082,federal_income_tax_before_refundable_credits,50,llm_error,taxable_income_or_deductions,False,,"The correct answer comes from the 2026 OBBBA rules for a head-of-household filer. AGI is $117,661. That counts wages net of only the 401(k) deferral (ESI premiums are not excluded), allows no IRA deduction because the head is an active 401(k) participant, and caps the capital loss at $3,000. The deductions are the $24,150 HoH standard deduction plus the new $700 charitable deduction for non-itemizers, giving taxable income of $92,811. Tax uses the 2026 HoH brackets (10% to $17,700, 12% to $67,450), with qualified dividends at 15%, minus a $600 CDCC and a $2,200 CTC. The largest group of near-misses landed at $9,734 because they skipped the $700 non-itemizer charitable deduction. Other models applied TCJA-sunset law (personal exemptions, $1,000 CTC), excluded the $21,208 ESI premiums from wages, used medical premiums to itemize, deducted the whole capital loss, or used pre-2026 bracket and deduction amounts."
+us,scenario_082,federal_income_tax_before_refundable_credits,51,llm_error,taxable_income_or_deductions,False,,"The reference uses 2026 post-OBBBA law. AGI is $117,661.12: the capital loss is capped at $3,000, the $68 refund is included, and there is no traditional IRA deduction because the head actively participates in a 401(k) and is above the phaseout. Deductions are the $24,150 head-of-household (HoH) standard deduction plus the new $700 charitable deduction for non-itemizers, leaving taxable income of $92,811.12. Ordinary income is taxed at HoH brackets of 10% to $17,700 and 12% to $67,450, then 22%; the $13,424 of qualified dividends is taxed at 15%; then the $2,200 CTC and $600 CDCC are subtracted. The closest misses either left out the $700 non-itemizer charitable deduction ($9,734) or took the $8 IRA deduction ($9,578.20). The larger misses assumed the TCJA had expired, counted employer-sponsored insurance (ESI) premiums as deductible or pre-tax, ignored the $3,000 capital-loss cap, or used outdated brackets and credit amounts."
us,scenario_082,federal_refundable_credits,8,llm_error,credit_phaseout,False,,"The separating step is that the Additional Child Tax Credit is a residual, not a per-child entitlement: the 2026 CTC of $2,200 is first applied nonrefundably against tax liability, and only the unused remainder — capped at $1,700 and further limited to 15% of earned income over $2,500 — becomes refundable. This head-of-household filer has roughly $117,650 of AGI (wages net of the $181 traditional 401(k) deferral, plus $20,480 of dividends and $100 of interest, less the $3,000 capital-loss limitation on a $30,711 net loss and the $8 IRA deduction), which produces federal tax before credits many times larger than $2,200, so the entire credit is consumed nonrefundably and nothing spills into the refundable bucket. EITC is zero on income grounds and independently on the investment-income limit, and no American Opportunity Credit, Recovery Rebate Credit, or refundable payroll tax credit applies. Every wrong model quoted a statutory refundable ceiling as the answer instead of computing the leftover after the nonrefundable CTC."
us,scenario_082,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_082,local_income_tax,6,llm_error,state_local_rule,False,,"The household facts give only state = NY and never say the household lives in New York City. The prompt says to treat any unlisted status input as false, so NYC residence is false. In New York, only NYC residence triggers the separate local income tax output, so local_income_tax is $0. Every wrong model assumed NYC residence and ran NYC resident brackets on NY taxable income, which turned a state-only fact pattern into an NYC tax bill."
-us,scenario_082,payroll_tax,46,prompt_ambiguity,state_local_rule,False,,"Federal employee FICA on the full $100,195 is $7,664.92: Social Security $6,212.09 plus Medicare $1,452.83, with no Additional Medicare Tax. The reference adds New York's employee contributions at their legal maximum: Paid Family Leave at 0.432% capped at $411.91, and disability benefits (DBL) at $0.60 a week, or $31.20. That brings the total to $8,108.03. ""Mandatory employee state payroll taxes"" can be read two ways: (1) NY PFL/DBL count at the maximum employee contribution, or (2) they are left out because a NY employer can choose to pay them instead of withholding. About 20 models took reading (2) and landed at or near $7,664.92. The other wrong answers came from three kinds of mistakes: using an older PFL rate or cap (0.388%, $354.53, $333), leaving the PFL uncapped ($432.84), or wrongly taking ESI premiums or 401(k) deferrals out of FICA wages. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 8,108.03; the alternative is 7,664.92. The prompt counts mandatory employee state payroll taxes; New York disability and Paid Family Leave contributions are the employer's, which may deduct the employee share from wages but need not (N.Y. Workers' Compensation Law ss. 209(3)-(4) and 210(1)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($443.11); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
+us,scenario_082,payroll_tax,47,prompt_ambiguity,state_local_rule,False,,"Federal employee FICA on the full $100,195 of wages is $6,212.09 + $1,452.83 = $7,664.92. Traditional 401(k) deferrals and the ESI premium input do not reduce this base, and no Additional Medicare Tax applies. The reference adds New York's maximum employee contributions on top of that: Paid Family Leave at 0.432%, capped at $411.91, and disability (DBL) at $0.60/week, or $31.20. That brings the total to $8,108.03. The prompt says 'mandatory employee state payroll taxes,' which can be read two ways: (a) New York's statutory maximum employee PFL/DBL withholding, which is the reference's reading, or (b) FICA only, because New York lets employers absorb those contributions. About 19 models chose reading (b). Most of the rest added New York contributions using stale or uncapped PFL rates, dropped DBL or PFL, or wrongly took ESI premiums or 401(k) deferrals out of FICA wages. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned prompt_ambiguity; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 8,108.03; the alternative is 7,664.92. The prompt counts mandatory employee state payroll taxes; New York disability and Paid Family Leave contributions are the employer's, which may deduct the employee share from wages but need not (N.Y. Workers' Compensation Law ss. 209(3)-(4) and 210(1)), and policyengine-us 2.15.17 counts the largest share the employer may deduct ($443.11); read as the amount the law requires of the employee, $0, the output is employee federal payroll tax alone. Found in the 2026-10-05 audit of state payroll components (reference_audit/2026-10-05-payroll, program_classification.json). Reference depends on an unlisted input; output excluded from scoring (whether the employer deducts the employee share of a state paid-leave or disability premium that the law lets it deduct but does not require, which decides whether that share is a mandatory employee state payroll tax)."
us,scenario_082,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_082,state_income_tax_before_refundable_credits,51,llm_error,thresholds_rates,False,,"Start with federal AGI of $117,661.12. That uses wages net of the traditional 401(k), allows only $3,000 of the capital loss, gives no deduction for the $8 IRA contribution, and does not subtract the ESI premiums. Subtracting the $67.50 refund gives NY AGI of $117,593.62. The NY head-of-household standard deduction of $11,200 beats itemizing ($9,328.22), and only one $1,000 dependent exemption applies, leaving $105,393.62 of taxable income. The 2026 HOH schedule uses rates of 3.9%/4.4%/5.15%/5.4% through $107,650, for $5,442.63. Because NY AGI is above $107,650, a $156.51 supplemental recapture tax applies. No nonrefundable credits apply: the NY CDCC and Empire State Child Credit are refundable, and the household credit is phased out. Most wrong answers went wrong in one of these ways: used pre-2026 rates, left out the supplemental tax, used a single filing status or the wrong deduction, subtracted the refundable CDCC, or removed the ESI premiums or the full capital loss from income."
-us,scenario_082,state_refundable_credits,51,llm_error,state_local_rule,False,,"The reference is the sum of two refundable New York credits under 2026 law, and nearly every model missed at least one of them. The first is the enhanced Empire State Child Credit: a $1,000 base for a child under 4, reduced by $16.50 per full $1,000 of federal AGI above the $75,000 head-of-household threshold (42 × $16.50 = $693), which leaves $307. The second is the new Tax Law 606(c-2) child and dependent care credit, which applies from 2026: about 29.352% of $3,000 in qualifying expenses at NY AGI of $117,593.62, or $880.55. It is not the old 20% of the $600 federal CDCC ($120). Models either used the pre-2025 ESCC formula (33% of federal CTC or a $100 minimum), applied the pre-2023 age floor or the wrong phase-out, or treated income as too high for any refundable credit."
+us,scenario_082,state_income_tax_before_refundable_credits,52,llm_error,thresholds_rates,False,,"New York tax here depends on four steps. First, subtract the $68 state refund from federal AGI ($117,661.12) to get NY AGI of $117,593.62. Second, file as head of household and take the $11,200 standard deduction plus one $1,000 dependent exemption, for taxable income of $105,393.62. Third, apply the reduced 2026 head-of-household rates (3.9%/4.4%/5.15%/5.4% at the $12,800/$17,650/$20,900 thresholds), which gives $5,442.63. Fourth, add the $156.51 supplemental tax (tax-benefit recapture) because NY AGI is above $107,650. No nonrefundable credit applies: the household credit is fully phased out, and the NY child and dependent care credit and the Empire State Child Credit are both refundable. Most wrong answers came from four mistakes: using the 2025 rates, leaving out the supplemental tax, choosing single status or the wrong deduction, or subtracting refundable credits."
+us,scenario_082,state_refundable_credits,52,llm_error,state_local_rule,False,,"The $1,187.55 reference is the sum of two refundable New York credits for 2026. The first is the enhanced Empire State Child Credit: a $1,000 base for a child under 4, minus a $693 phase-out at federal AGI of about $117,661, leaving $307. The second is the child and dependent care credit under the new Tax Law 606(c-2), which applies from 2026. It applies an applicable percentage (about 29.35% at NY AGI of about $117,594) directly to the $3,000 one-child expense cap, giving $880.55. Almost every model either applied the old care-credit rule (20% of the $600 federal credit, or $120) or left the care credit out. Most also got the child credit wrong by using the old 33%-of-federal-CTC or $100-minimum formulas, wrongly excluding children under 4, or skipping or misstating the phase-out."
us,scenario_083,federal_income_tax_before_refundable_credits,1,llm_error,thresholds_rates,False,,"The 2026 single-filer standard deduction exceeds the household’s $11,010 AGI, leaving zero taxable income. With no other federal tax components or nonrefundable credits affecting the result, federal income tax before refundable credits is $0."
us,scenario_083,federal_refundable_credits,9,llm_error,categorical_eligibility,False,,"A childless filer must satisfy the EITC age requirement, and the 20-year-old head does not. With no qualifying children and no applicable refundable education, recovery rebate, or payroll tax credit, every component of federal refundable credits is zero."
us,scenario_083,head_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Texas did not adopt ACA Medicaid expansion, so income below 138% of FPL does not create Medicaid eligibility for a nondependent adult. The 20-year-old head has no qualifying Texas category—such as pregnancy, parent/caretaker status, disability, or SSI eligibility—so `medicaid_category` is NONE regardless of the 0.69 FPL MAGI level."
@@ -667,7 +667,7 @@ us,scenario_083,self_employment_tax,1,llm_error,payroll_tax_base,False,,"Ordinar
us,scenario_083,snap,18,llm_error,asset_resource,False,,"This one-person Texas household passes every SNAP income test: gross income is $917.50 a month, well under 130% and 165% of FPL, and net income is about $591, under 100%. What makes it ineligible is its resources. Texas's broad-based categorical eligibility keeps a $5,000 asset limit, and the federal standard for a non-elderly household is about $3,000. The head's $6,620 bank balance exceeds both, so SNAP is $0. Every model ran the income-based benefit formula, getting about $110-$245 a month. None applied the resource test, and several said outright that Texas BBCE waives the asset test."
us,scenario_084,head_medicaid_eligible,6,llm_error,categorical_eligibility,False,,"Disability, blindness, SSDI receipt, and qualifying immigration status do not independently establish Medicaid eligibility; the person must satisfy a specific North Carolina coverage pathway and its financial rules. The head qualifies through no Medicaid category, and Medicaid MAGI is 1.56 times FPL, above the expansion threshold."
us,scenario_084,head_medicare_eligible,37,prompt_ambiguity,age_disability,False,,"PolicyEngine grants eligibility at age 65 or older, or with positive Social Security disability income and at least 24 months of receipt. The head is 62 and has SSDI income; months of receipt are unstated. The frozen value is 0.0, and the output is excluded from scoring because the household facts do not settle whether the duration requirement is met. Every model treated disability status plus current SSDI income as enough for the disability pathway, and many assumed the 24-month wait had already passed. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 0.0 to 1.0 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. In law the under-65 route needs 24 months of SSDI entitlement; a year-round recipient has at least 12. The facts do not settle it."
-us,scenario_084,snap,2,llm_error,thresholds_rates,False,,"The single-person household has $33,233 of gross annual income ($2,769/month), which exceeds North Carolina's broad-based categorical eligibility gross cap of 200% FPL (about $2,680/month for one person in 2026), and after the ~$204 standard deduction and the elderly/disabled excess-medical deduction its net income of roughly $1,738/month still exceeds the 100%-FPL net income limit of about $1,340/month; the $56,500 bank balance also exceeds the elderly/disabled resource limit of roughly $4,650. Ineligibility ends the calculation at $0. Both models stopped at the observation that the gross income test is waived when an elderly or disabled member is present and never tested net income against the 100%-FPL ceiling, then manufactured a benefit — one by overriding its own $0 formula result, the other by applying the one-to-two-person minimum allotment, which is a floor for households already found eligible, not a substitute for eligibility."
+us,scenario_084,snap,3,llm_error,thresholds_rates,False,,"SNAP counts all unearned income: $8,400 in disability benefits, $24,828 in SSDI and $5 in interest, for $33,233 a year, or about $2,769 a month. That is above NC's broad-based categorical eligibility limit of 200% FPL, about $2,608 a month for one person. Without categorical eligibility, the elderly/disabled household must pass the regular net income test (100% FPL, about $1,304 a month) and the $4,500 asset limit. Net income stays at about $1,333 or more even after the standard and excess medical deductions, and the $56,500 in bank assets is far over the limit, so SNAP is $0. All three models paid a positive benefit (a minimum allotment or a made-up amount) without first confirming the household was eligible."
us,scenario_085,federal_income_tax_before_refundable_credits,37,llm_error,taxable_income_or_deductions,False,,"This is a single filer, age 67. AGI is $45,573.74, which includes $14,050.74 of taxable Social Security: provisional income is $45,236, and the taxable amount is $4,500 plus 85% of the excess over $34,000. Under OBBBA's 2026 permanent-TCJA law, the deduction is $24,150: the $16,100 single standard deduction, the $2,050 aged addition, and the $6,000 senior deduction, which does not phase out at this MAGI. Tax on $21,423.74 of taxable income at 10% up to $12,400 and 12% above is $2,322.85, and no nonrefundable credits apply. Most wrong answers either left out the $6,000 senior deduction, applied a TCJA-sunset regime with personal exemptions and a 15% bracket, miscomputed the taxable share of Social Security, or submitted a number that contradicts their own correct reasoning."
us,scenario_085,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_085,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -677,12 +677,12 @@ us,scenario_085,ssi,1,parse_contract_failure,missing_output,False,,All wrong res
us,scenario_085,state_income_tax_before_refundable_credits,37,llm_error,state_local_rule,False,,"For a 67-year-old, Pennsylvania exempts Social Security survivor benefits and the pension, IRA and 401(k) distributions. That leaves $1,675 of interest, which gives a gross tax of $51.42 at the 3.07% rate. Pennsylvania Tax Forgiveness (Schedule SP) is a nonrefundable credit. Its eligibility income leaves out Social Security and exempt retirement distributions, so eligibility income is only $1,675. That is far below the $6,500 single-filer limit for 100% forgiveness, which wipes out the $51.42 and brings tax before refundable credits to $0. Most models stopped at $51.42, either by skipping forgiveness or by wrongly counting Social Security and pensions in eligibility income. A few also taxed retirement distributions that are exempt at age 67."
us,scenario_086,child1_chip_eligible,1,llm_error,categorical_eligibility,False,,"CHIP requires Medicaid ineligibility and satisfaction of CHIP’s independent age and income criteria. Child 1 is age 12, so age is not the failing criterion; the household income exceeds Georgia’s applicable CHIP income threshold for 2026."
us,scenario_086,child1_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"The models conflated Georgia’s separate PeachCare/CHIP pathway with Medicaid or assumed that being a child creates Medicaid eligibility at higher income levels. At 3.44 times FPL, child1 exceeds Georgia’s child Medicaid thresholds and qualifies for no Medicaid category, so employer-sponsored coverage and non-MAGI expense deductions do not create eligibility."
-us,scenario_086,federal_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"The reference counts the $12,824 of Social Security dependent benefits as the head's income. Provisional income of about $68,075 is far above the $34,000 head-of-household upper base, so 85% ($10,900.40) is taxable. The full $337.50 educator expense is deducted, and no auto-loan-interest deduction, mortgage interest, or personal exemptions are allowed. That gives AGI of $72,562.90 and taxable income of $48,412.90 after the 2026 head-of-household standard deduction of $24,150. Tax at 10% up to $17,700 and 12% above that is $5,455.55, and the full $2,200 OBBBA child tax credit is used as a nonrefundable credit. Models missed the answer mainly by leaving out the taxable Social Security, applying TCJA-sunset rules (personal exemptions, 15% bracket, $1,000 CTC), using the wrong filing status, using 2025 or pre-OBBBA parameters, or deducting auto loan interest. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The 2026 educator expense cap is $350, so the head's $337.50 is fully deductible; the judge's $300 cap is not the law. Reference affirmed (26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12)."
+us,scenario_086,federal_income_tax_before_refundable_credits,48,llm_error,taxable_income_or_deductions,False,,"The correct computation treats the $12,824 of Social Security benefits as the Head's income. Because provisional income is far above the $34,000 head-of-household threshold, 85% of the benefits ($10,900.40) is taxable. The full $337.50 educator expense is deductible, so AGI is $72,562.90. Subtracting only the OBBBA 2026 head-of-household standard deduction of $24,150 (no auto-loan interest deduction and no personal exemptions) gives taxable income of $48,412.90. The 2026 head-of-household brackets (10% up to $17,700, then 12%) produce $5,455.55 of tax, and the full $2,200 CTC is nonrefundable against it, leaving $3,255.55. Models mostly went wrong in five ways: they excluded the benefits as the child's income, took a $275 car-loan interest deduction, capped the educator deduction at $300, used stale or pre-OBBBA standard deductions and CTC amounts, or assumed the TCJA would sunset. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (taxable_income_or_deductions). The 2026 educator expense cap is $350, so the head's $337.50 is fully deductible; the judge's $300 cap is not the law. Reference affirmed (26 U.S.C. 62(a)(2)(D), 62(d)(3); Rev. Proc. 2025-32 section 3.12)."
us,scenario_086,federal_refundable_credits,11,llm_error,credit_phaseout,False,,"The 2026 Child Tax Credit for one child is $2,200. The refundable Additional CTC equals only the part of that credit left over after it offsets income tax, capped at $1,700 and at 15% of earnings over $2,500. This Head of Household filer has $62,000 in wages, so taxable income is at least about $37,500 after the $24,150 standard deduction. That gives pre-credit tax of about $4,100 or more, which absorbs the whole $2,200 credit and leaves $0 of ACTC. Earnings of $62,000 are also above the one-child EITC phase-out end of about $51,000, so EITC is $0. Most models treated the $1,700 refundability cap as the amount payable and skipped the step that limits ACTC to unused credit after tax liability."
us,scenario_086,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_086,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_086,payroll_tax,8,llm_error,payroll_tax_base,False,,"Employee payroll tax is $3,844 of Social Security tax plus $899 of Medicare tax on the head’s full $62,000 wage amount, totaling $4,743. The listed insurance premiums do not reduce the FICA wage base in this computation, Georgia unemployment insurance is employer-paid rather than a mandatory employee payroll tax, and the dependent child has no wages."
-us,scenario_086,state_income_tax_before_refundable_credits,50,llm_error,thresholds_rates,False,,"Georgia's 2026 computation starts from GA AGI of $61,662.50: federal AGI of $72,562.90 minus the $10,900.40 of taxable Social Security, which Georgia excludes. It then subtracts the $15,000 head-of-household standard deduction and a $5,000 dependent exemption, which leaves $41,662.50 of taxable income taxed at the flat 4.99% rate, for $2,078.96. Most models used older or projected parameters: a $12,000, $18,500 or $24,000 standard deduction, a $3,000 or $4,000 dependent exemption, a personal exemption that no longer exists, and flat rates of 5.09%–5.39%. Some also mishandled the Social Security subtraction, either keeping it in income or subtracting it twice."
+us,scenario_086,state_income_tax_before_refundable_credits,51,llm_error,thresholds_rates,False,,"Georgia AGI is federal AGI ($72,562.90) minus the fully exempt taxable Social Security ($10,900.40), which gives $61,662.50. The 2026 Georgia parameters are a $15,000 head-of-household standard deduction, a $5,000 dependent exemption and a 4.99% flat rate. Applying them gives taxable income of $41,662.50 × 4.99% = $2,078.96. Almost every model used outdated or invented parameters: rates of 5.09–5.49%, a $12,000/$18,500/$24,000 standard deduction, or a $3,000/$4,000 dependent exemption, sometimes with a personal exemption that no longer exists. Some also left Social Security in Georgia income or subtracted employer insurance premiums or auto-loan interest."
us,scenario_086,state_refundable_credits,1,llm_error,categorical_eligibility,False,,"Georgia does not provide the claimed $260 refundable credit for each dependent child under age 17. With no applicable Georgia refundable credit, the state-level refundable-credit aggregate is $0."
us,scenario_088,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"The decisive computation step is the 2026 deduction for qualified tip income: the $9,600 of tips included in wages reduces taxable income in addition to the applicable standard deduction. The state income-tax refund is also excluded because no prior-year itemized-deduction tax benefit is established. These deductions reduce taxable income to zero, so federal income tax before refundable credits is $0 and cannot be negative."
us,scenario_088,federal_refundable_credits,7,llm_error,credit_phaseout,False,,"The only refundable credit in play is the childless EITC, and IRC §32(a)(2)(B) phases it out against the GREATER of earned income or AGI. Earned income is $18,577, but AGI is $24,035 once the $2,733 unemployment compensation, $1,862 taxable pension, $675 state tax refund, $140 interest and $48 dividends are added; the tip deduction is below-the-line and does not reduce AGI. The 2026 childless EITC completes its phase-out below $20,000, so an AGI of $24,035 zeroes the credit, and with no qualifying children, no education expenses and no rebate, every other refundable component is $0. Every wrong model ran the phase-out on earned income alone, or on a max-credit shortcut, and never applied the AGI leg of the test."
@@ -704,12 +704,12 @@ us,scenario_090,snap,8,llm_error,asset_resource,False,,"Kansas does not use broa
us,scenario_090,state_income_tax_before_refundable_credits,22,llm_error,taxable_income_or_deductions,False,,"Kansas SB 1 (2024) raised the single filer's personal exemption from $2,250 to $9,160, starting in tax year 2024. With the single standard deduction of about $3,605, the two subtractions total about $12,765. That is far more than this filer's $7,020 Kansas AGI, so Kansas taxable income is $0 and tax is $0. Every model used the old $2,250 exemption, the $2,320 dependent-exemption amount, or no exemption at all. Each one therefore left a positive taxable balance and taxed it at 3.1%, 5.2%, or an invented rate."
us,scenario_090,state_refundable_credits,5,llm_error,categorical_eligibility,False,,"Kansas's refundable individual credit is its EITC, set at 17% of the federal EITC, and the head is 22 — below the age-25 floor for the childless federal EITC — so the federal credit is $0 and the state match on it is $0. The other candidate, the Kansas food sales tax credit, is nonrefundable for tax years after 2012 and in any case requires the filer to be 55 or older, blind or disabled, or to have a dependent child under 18 in the home, none of which describes a 22-year-old single filer with no dependents. Every wrong answer cleared a household income test ($7,020 is well under both credits' ceilings) and then stopped, never applying the categorical age/dependent gate that actually decides both credits. The correct derivation is 17% × $0 federal EITC plus $0 qualifying food sales tax credit = $0."
us,scenario_090,tanf,1,llm_error,categorical_eligibility,False,,"TANF is a benefit for families with dependent children; a single 22-year-old with no children fails Kansas's categorical requirement outright, and the $20,300 in bank assets independently exceeds the state's TANF resource limit, so the correct answer is $0. The trap is the ""financial assistance"" line item in the household facts: it is an unearned-income input, not a TANF payment, and treating it as the answer skips the eligibility determination entirely. Answering this variable requires running the categorical test — dependent child present, resources under the limit — rather than pattern-matching a dollar figure labeled ""assistance"" in the inputs."
-us,scenario_091,federal_income_tax_before_refundable_credits,47,llm_error,taxable_income_or_deductions,False,,"To reach the reference, a model needs four 2026 rules under the OBBBA. First, use the $16,100 single standard deduction with TCJA rates made permanent, so no personal exemption and no 15% bracket. Second, deduct the $628 FLSA overtime premium as the new qualified-overtime deduction. Third, tax the $3,968 of qualified dividends and $1,170 of capital-gain distributions at 0%, which leaves $14,316.66 of ordinary taxable income and $1,470 of tax (10% up to $12,400, then 12%). Fourth, subtract a $120 Saver's Credit (10% of $1,199 in traditional and Roth 401(k)/IRA contributions); the AOTC is zero because no qualified tuition expenses are listed. Most wrong answers fall into one of these groups: they skipped the overtime deduction, used a pre-2026 standard deduction or bracket, left out the Saver's Credit, claimed an AOTC with no tuition, subtracted ESI premiums from wages, or assumed the TCJA sunset."
+us,scenario_091,federal_income_tax_before_refundable_credits,48,llm_error,taxable_income_or_deductions,False,,"Getting to the reference takes four steps. First, take the $628 qualified-overtime deduction for the FLSA premium. Second, use the 2026 single standard deduction of $16,100, not a 2025 figure or the pre-TCJA amount. Third, put qualified dividends and capital gains in the 0% band, which leaves $14,317 of ordinary taxable income and $1,470 of tax under the 2026 10%/12% brackets. Fourth, subtract the 10% nonrefundable saver's credit on $1,199 of traditional and Roth 401(k)/IRA contributions (≈$120) to get $1,350. The common errors were leaving out the overtime deduction, using an outdated standard deduction, claiming an AOTC with no qualified tuition listed, subtracting ESI premiums from wages that were already stated as gross, and stopping at $1,470 without the saver's credit."
us,scenario_091,federal_refundable_credits,10,llm_error,other,False,,"The decisive step is the American Opportunity Credit's base: the credit is 100% of the first $2,000 plus 25% of the next $2,000 of qualified tuition and related expenses, and this household lists no tuition — an unlisted numeric input the prompt explicitly instructs be treated as $0 — so the credit is $0 and its 40% refundable portion is $0. The listed AOTC items (eligible institution, 1098-T/exception, institution EIN, at least half-time, pursuing a credential) are administrative gating conditions, not expenditures, and nine of ten models converted those flags into the statutory maximum $2,500 credit and booked $1,000 (or a mis-scaled $400/$1,200) of refundability. The EITC is separately $0 three times over: the head is 22, below the age-25 floor for a childless filer, and both the ~$36,000 AGI and the childless phase-out ceiling foreclose it; with no children there is no refundable CTC, no recovery rebate credit and no refundable payroll tax credit, leaving $0."
us,scenario_091,head_medicaid_eligible,1,llm_error,thresholds_rates,False,,"Wisconsin Medicaid eligibility requires qualification through an applicable categorical pathway and its income rules. This 22-year-old nondependent falls into no Medicaid category, and MAGI of 2.19 times FPL does not satisfy an adult eligibility pathway, so immigration eligibility alone does not confer coverage."
-us,scenario_091,payroll_tax,17,llm_error,payroll_tax_base,False,,"Employee payroll tax here is 6.2% Social Security plus 1.45% Medicare on the full $27,000 of gross wages, which comes to $1,674.00 + $391.50 = $2,065.50. The listed $6,589 of employer-sponsored insurance premiums and the $1,800 of other health premiums do not lower the FICA wage base. The $628 FLSA overtime premium is already inside the $27,000 gross, and Wisconsin has no mandatory employee state payroll tax. Most of the wrong models treated the ESI premiums as a pre-tax Section 125 deduction and taxed only $20,411. The rest added the overtime twice, invented a state payroll tax, used the wrong combined rate, or made an arithmetic or reporting slip."
+us,scenario_091,payroll_tax,18,llm_error,payroll_tax_base,False,,"PolicyEngine applies the 6.2% Social Security rate and the 1.45% Medicare rate to the full $27,000 of gross wages. That gives $1,674.00 + $391.50 = $2,065.50. It does not reduce the wage base for employer-sponsored insurance premiums, other health premiums, or traditional 401(k) deferrals. The $628 overtime premium is already included in gross wages, and Wisconsin has no mandatory employee state payroll tax. Most wrong answers came from cutting the FICA base for the $6,589 ESI premiums, which gives $1,561.44. The rest came from other wage-base changes, a wrong rate, an invented state tax, arithmetic slips, or a submitted value that differed from the model's own explanation."
us,scenario_091,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_091,state_income_tax_before_refundable_credits,51,llm_error,taxable_income_or_deductions,False,,"Wisconsin tax here depends on four state-specific steps. First, Wisconsin excludes 30% of the $1,170 capital gain distributions ($351), so federal AGI of $36,182.56 becomes Wisconsin AGI of $35,831.56. Second, the sliding-scale single standard deduction at that income is $12,074.61: it is neither fully phased out nor left at its unreduced maximum. Third, the $700 exemption is a deduction, not a credit. Fourth, the 2026 rates are 3.5% up to about $15,110 and 4.4% above it. Wisconsin taxable income is $23,056.95 and no Wisconsin nonrefundable credit applies, so tax is $878.52. The models went wrong in five main ways: they skipped the 30% exclusion, sized the standard deduction wrongly, dropped the exemption, subtracted ESI premiums, overtime or health premiums that the engine does not subtract in this reference, or used the federal AOTC or credits that don't exist to bring the tax to zero."
+us,scenario_091,state_income_tax_before_refundable_credits,52,llm_error,state_local_rule,False,,"The reference starts from federal AGI of $36,182.56. Wages are reduced only by the $926 traditional 401(k) deferral, not by ESI premiums or overtime, and the $43 IRA deduction is subtracted. Wisconsin then subtracts 30% of the $1,170 capital gain distributions ($351), and the 2026 sliding-scale single standard deduction phases down only to $12,074.61, followed by the $700 exemption. That leaves $23,056.95 of taxable income, taxed at 3.5% up to the indexed first bracket of about $15,110 and 4.4% above it, for $878.52. No Wisconsin nonrefundable credit applies, because there is no rent, property tax, or Wisconsin AOTC analog. Most models missed the $351 exclusion, mis-sized the standard-deduction phase-down (some phased it out entirely, others left it unphased), dropped the $700 exemption, or wrongly cut income for ESI premiums, health premiums, or overtime. A few zeroed the tax with credits that do not exist or do not apply."
us,scenario_091,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_092,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"Only the $14,980 taxable pension enters adjusted gross income; the Social Security benefit contributes no taxable amount under the federal provisional-income calculation. The age-77 single filer’s applicable standard deduction exceeds that AGI, leaving zero taxable income and therefore zero federal income tax before refundable credits."
us,scenario_092,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Age 77 and disability do not independently establish Alabama Medicaid eligibility; the person must qualify through a specific Medicaid category. This person receives no SSI, has MAGI income of 2.03 times FPL, and qualifies through none of the available pathways, so the Medicaid category is NONE and eligibility is No."
@@ -719,10 +719,10 @@ us,scenario_092,state_income_tax_before_refundable_credits,25,llm_error,state_lo
us,scenario_092,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_093,dependent1_medicaid_eligible,1,llm_error,categorical_eligibility,False,,"Disability alone does not confer Missouri Medicaid eligibility. The person must satisfy a specific non-MAGI pathway's categorical and financial requirements; this 27-year-old dependent receives no SSI, qualifies for no non-MAGI category, and at 2.82 times FPL exceeds the applicable MAGI limits."
us,scenario_093,dependent1_medicare_eligible,5,llm_error,age_disability,False,,"PolicyEngine's is_medicare_eligible turns on a single age test — age 65 or older — with the narrow ESRD/ALS entitlements not modeled and no disability branch at all; Dependent 1 is 27, so the variable is False. Every wrong model substituted the household's bare ""is disabled"" flag for that age test, asserting a disability-based Medicare pathway either as a PolicyEngine rule or as a real-world shortcut. The statutory disability pathway requires 24 months of SSDI entitlement, and Dependent 1 has $45,000 in annual wages (~$3,750/month), far above substantial gainful activity, plus no listed SSDI receipt — the prompt directs unlisted benefit receipt to be treated as zero, so no disability pathway is reachable on these facts."
-us,scenario_093,federal_income_tax_before_refundable_credits,50,prompt_ambiguity,household_unit_or_filing_status,False,,"The tax unit is Head and Spouse filing jointly, with the disabled 27-year-old as their dependent. The engine grants the $500 credit for the listed adult tax-unit dependent. Their $45,000 of wages is excluded from the joint return's gross income. The frozen reference includes only the joint return; the output is excluded because the prompt leaves the inclusion of the dependent's separate return unresolved. Under 2026 law, wages of $113,942 after 401(k) deferrals plus $1,156 of interest, minus the $2,500 student-loan cap and the IRA deduction, give AGI of $112,408. Subtracting the $32,200 standard deduction leaves taxable income of $80,208. Tax at 10% up to $24,800 and 12% above that is $9,129, and the $500 credit brings it to $8,629. Most models went wrong in one or more of these ways: they added the dependent's wages to the joint return, denied dependency and so lost the $500 credit, subtracted ESI premiums from wages, took a $1,000 auto-loan-interest deduction, or used pre-OBBBA or TCJA-sunset parameters. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 8,628.99; the alternative is 11,848.99. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
+us,scenario_093,federal_income_tax_before_refundable_credits,51,prompt_ambiguity,household_unit_or_filing_status,False,,"The reference treats the household as one joint return. The 27-year-old is permanently disabled, so they count as a qualifying child at any age. They are claimed for the $500 credit for other dependents, and their $45,000 of wages stays off the parents' return with no separate return filed. AGI is $112,408: wages of $118,000, less $4,058 of traditional 401(k) deferrals, plus $1,156 of interest, less $190 of IRA contributions and the $2,500 student-loan-interest cap. Subtracting the 2026 OBBBA standard deduction of $32,200 leaves $80,208 taxable. At the 10% rate up to $24,800 and 12% above it, tax is $9,128.99, and the $500 credit brings it to $8,628.99. ESI premiums are not subtracted from wages, and no auto-loan-interest deduction applies. Most models failed in one of three ways: they added the dependent's wages to the joint return, filed a separate return for the dependent and added that tax, or refused to claim the dependent because of the qualifying-relative gross-income test. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 8,628.99; the alternative is 11,848.99. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
us,scenario_093,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_093,payroll_tax,19,llm_error,payroll_tax_base,False,,"Employee FICA here is 7.65% of each worker's full listed gross wages: 6.2% Social Security plus 1.45% Medicare on $53,000, $65,000 and $45,000, all below the wage base. That gives $3,286 + $768.50, $4,030 + $942.50 and $2,790 + $652.50, for a total of $12,469.50. No Additional Medicare Tax applies, and Missouri has no employee state payroll tax. The facts never say the $8,389 ESI premiums were paid through a pre-tax cafeteria plan. The largest group of models (8) assumed a Section 125 salary reduction anyway, cutting FICA wages to $137,833 and the tax to $10,544.22. Most other models wrote out the correct $12,469.50 components or formula but then added wrong, multiplied wrong, or submitted a different number from the one they computed."
-us,scenario_093,state_income_tax_before_refundable_credits,51,prompt_ambiguity,household_unit_or_filing_status,False,,"The main trap is the tax unit. The frozen reference covers the couple's return. The adjudication establishes that the dependent must file a separate federal and state return and excludes this output because household scope is unresolved. Many models added that wage income to joint AGI, which pushed AGI above $125,000 and eliminated the federal-tax deduction. Others added a separate single-filer tax for the dependent. For the couple's return, the engine uses a $431.45 federal-tax deduction, the $32,200 standard deduction, and the 2026 brackets on each spouse's proportional share ($34,743.94 and $45,032.86). The 5% federal-tax deduction rate applies to AGI of at least $100,000 and below $125,000. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 3,388.25; the alternative is 4,528.08. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
+us,scenario_093,payroll_tax,20,llm_error,payroll_tax_base,False,,"The reference is employee FICA at 7.65% (6.2% Social Security plus 1.45% Medicare) on the full gross wages of $53,000, $65,000 and $45,000, which total $163,000. That gives $10,106 + $2,363.50 = $12,469.50. Every wage is below the Social Security wage base, combined wages are below the Additional Medicare Tax threshold, and Missouri has no employee payroll tax. The most common error was treating the $8,389 employer-sponsored insurance premium as a Section 125 salary reduction and subtracting it from FICA wages; the prompt gives no such fact, so wages are $137,833 too low by that method. One model wrongly subtracted traditional 401(k) deferrals, which are FICA-taxable. Most of the other models wrote down the correct $12,469.50 or the correct formula, then made an arithmetic slip or submitted a different number."
+us,scenario_093,state_income_tax_before_refundable_credits,52,prompt_ambiguity,household_unit_or_filing_status,False,,"The disabled adult dependent is part of the couple's tax unit and has $0 Missouri AGI, so the dependent's $45,000 of wages is not taxed. That leaves only the spouses' MO AGI of $112,408.25, which falls in the $100k–$125k band where Missouri's federal income tax deduction is 5% of the couple's $8,629 federal liability ($431.45). After the $32,200 standard deduction, MO taxable income is $79,776.80. On the combined return, that amount is split by each spouse's share of income ($34,743.94 and $45,032.86), and each share is taxed separately on the 2026 schedule (top rate 4.7%), so the low brackets apply twice and the total is $3,388.25. Most models either added the dependent's wages to AGI, which pushed it above $125k and removed the federal-tax deduction, or filed a separate return for the dependent. Most of the models that got the spouses' base right then skipped the 5% federal-tax deduction and ran the combined income through the brackets only once. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 3,388.25; the alternative is 4,528.08. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
us,scenario_095,federal_income_tax_before_refundable_credits,3,llm_error,household_unit_or_filing_status,False,,"The head's surviving-spouse status determines the federal filing-status pathway and supplies a standard deduction large enough to eliminate the $15,286.89 of AGI. Social Security is not taxable at this provisional-income level, so it is excluded from AGI; after the half-self-employment-tax deduction, taxable income and income tax before credits are both zero."
us,scenario_095,federal_refundable_credits,1,llm_error,categorical_eligibility,False,,"Federal refundable credits include only the refundable credit components applicable under 2026 law. Age 73 does not independently create a $1,200 refundable senior credit, and all five applicable components—EITC, refundable American Opportunity Credit, refundable CTC, Recovery Rebate Credit, and refundable payroll tax credit—equal zero."
us,scenario_095,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -735,7 +735,7 @@ us,scenario_095,reduced_price_school_meals_eligible,1,parse_contract_failure,mis
us,scenario_095,self_employment_tax,7,llm_error,payroll_tax_base,False,,"The correct method has two steps. First, reduce self-employment income to net earnings from self-employment by multiplying by 0.9235, which gives about $3,877. That amount is above the $400 filing floor and well below the Social Security wage base. Second, apply the full 15.3% rate, which gives about $593. Some models left out the 0.9235 adjustment. Others compared the wrong figure to the $400 floor or treated the income-tax deduction for half of SE tax as a reduction of SE tax itself. Others did the arithmetic and then submitted a different number. The head's age of 73 and receipt of Social Security do not exempt them from SE tax."
us,scenario_095,snap,5,llm_error,thresholds_rates,False,,"The household’s $29,605 annual income exceeds the applicable 2026 New Jersey SNAP gross-income eligibility limit for one person, so the SNAP calculation stops with no eligibility and a $0 allotment. Broad-based categorical eligibility, elderly-household deductions, and the minimum-allotment rule do not produce a benefit after this income screen fails."
us,scenario_095,state_income_tax_before_refundable_credits,24,llm_error,state_local_rule,False,,"New Jersey has a low-income rule (N.J.S.A. 54A:2-4): no Gross Income Tax is owed when NJ gross income is at or below $10,000 for single filers, or $20,000 for joint, head-of-household, and qualifying surviving-spouse filers. Social Security is excluded and the 62+ pension exclusion removes the $11,385 401(k) distribution, so NJ gross income is only $4,198 of self-employment profit. That is below even the $10,000 single threshold, and the pre-exclusion total of $15,583 is also below $20,000, so NJ tax is $0. Every model that answered applied exemptions and the 1.4% rate to the leftover self-employment income and never checked the gross-income threshold that zeroes the tax."
-us,scenario_095,state_refundable_credits,4,llm_error,credit_phaseout,False,,"This household has no children, so the NJ EITC is its only possible refundable credit. For a childless filer aged 65 or older, New Jersey computes the credit as 40% of a hypothetical federal childless EITC. The phase-in gives 7.65% × earned income of about $3,901 ($4,198 SE income minus $296.58, the deductible half of SE tax), or $298.45. The phase-out then takes 7.65% × (AGI − non-joint threshold). AGI is about $15,286 because it includes the $11,385 401(k) distribution (Social Security is untaxed because provisional income is below $25,000). Since the 401(k) distribution alone exceeds the roughly $11k threshold, the reduction is larger than the phase-in amount and the credit is $0. The household paid no property tax or rent, so the NJ property tax credit is also $0, and ANCHOR is not an income tax credit."
+us,scenario_095,state_refundable_credits,5,llm_error,age_disability,False,,"This NJ household has a 73-year-old head with no children and no property taxes or rent, so the only possible refundable state credit is the NJ EITC. In PolicyEngine, the NJ EITC is 40% of the federal EITC. The federal childless EITC is $0 because the head is over the age-64 limit, so nj_refundable_credits is $0. Three models assumed NJ drops the upper age limit for childless filers and got $119.38 (40% of 7.65% of net SE earnings). One model counted ANCHOR property-tax relief as an income-tax credit, and one model gave no answer."
us,scenario_095,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_098,federal_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"The $11,100 farm rental loss offsets the $15,540 taxable pension, leaving $4,440 of non-Social-Security income. Provisional income is then below the threshold for taxing Social Security benefits, so AGI remains $4,440; the applicable standard deduction eliminates taxable income and produces zero federal income tax before refundable credits."
us,scenario_098,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -761,15 +761,15 @@ us,scenario_099,child2_head_start_eligible,2,llm_error,categorical_eligibility,F
us,scenario_099,child2_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,child2_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,child2_wic_eligible,4,llm_error,categorical_eligibility,False,,"WIC eligibility requires both categorical eligibility and financial eligibility; being age four satisfies only the child-category requirement. The household's approximately $159,676 of annual income exceeds California's WIC limit of 185% of the federal poverty level for a four-person household, so child 2 is not eligible."
-us,scenario_099,federal_income_tax_before_refundable_credits,50,llm_error,credit_phaseout,False,,"The reference applies OBBBA rules for 2026: a $32,200 MFJ standard deduction, the new $2,000 MFJ charitable deduction for non-itemizers, a $2,200 CTC per child, and the enhanced CDCC. The CDCC rate falls from 35% by 1 point per $4,000 of joint AGI above $150,000, which gives 32% on $6,000 ($1,920) at an AGI of $159,729. Tax before credits is $17,031.48. The traditional IRA contributions are not deductible because of the active-participant phaseout, and no auto loan interest deduction applies. Most models used the old 20% CDCC rate, left out the $2,000 non-itemizer charitable deduction, used pre-OBBBA CTC or standard deduction amounts, or applied TCJA-sunset law with personal exemptions and a $1,000 CTC."
+us,scenario_099,federal_income_tax_before_refundable_credits,51,llm_error,credit_phaseout,False,,"For 2026 the OBBBA rules apply. The standard deduction is $32,200 MFJ, and non-itemizers also get a $2,000 charitable deduction, so taxable income is $125,529.30. Pre-credit tax is $17,031.48. The CTC is $2,200 per child and fully nonrefundable against this liability, for $4,400. The enhanced CDCC rate steps down from 35% by one point per $4,000 of AGI over $150,000, giving 32% at AGI $159,729 and a $1,920 credit; the traditional IRA is nondeductible because of the active-participant phaseout, and there is no auto-loan-interest or overtime deduction. Most wrong answers used the old 20% CDCC rate, left out the $2,000 non-itemizer charitable deduction, used the $2,000 pre-OBBBA CTC, or assumed TCJA expired (personal exemptions and the old 15%/25% brackets)."
us,scenario_099,federal_refundable_credits,6,llm_error,credit_phaseout,False,,"The trap is §24(d)'s liability-limitation prong: the refundable CTC is the *lesser* of the earned-income formula (15% of earnings over $2,500), the per-child refundable cap, and the portion of the CTC left unused after the credit offsets tax liability. This married-filing-jointly household has AGI near $159,676 (wages $165,000 plus $1,010 interest, $144 dividends, capital netting limited to a $3,000 loss, less $3,232 of traditional 401(k)/IRA contributions and the $338 educator expense) and roughly $127,000 of taxable income after the 2026 MFJ standard deduction, producing about $17,400 of federal income tax before credits — far more than the $4,400 of 2026 OBBBA CTC ($2,200 × 2), so the entire CTC is consumed nonrefundably and nothing spills into the ACTC. EITC is zero because $159,676 is far beyond the two-child MFJ phase-out, and the CDCC is nonrefundable. Every wrong model treated a refundability cap or the earned-income formula as a standalone entitlement instead of applying the unused-credit ceiling."
-us,scenario_099,free_school_meals_eligible,43,llm_error,state_local_rule,False,,"California runs a universal school meals program (state law since the 2022-23 school year). Under it, every enrolled K-12 student gets free breakfast and lunch whatever the household income. PolicyEngine therefore sets school_meal_tier to FREE for this household, even though its income is 5.03 times the federal poverty guideline and it has no SNAP/TANF categorical eligibility. That yields $1,119.60 of free-meal support for the 9-year-old in K-12. Every model that answered treated an income limit as decisive and never applied California's universal-meals override."
+us,scenario_099,free_school_meals_eligible,44,llm_error,state_local_rule,False,,"California runs a universal free school meals program (the California Universal Meals Program, in effect since the 2022-23 school year). Every enrolled K-12 student gets free meals regardless of household income. PolicyEngine sets school_meal_tier to FREE for California households, so the 9-year-old K-12 child receives about $1,131 a year and the household is eligible even though its income is 5.03 times the poverty guideline and it has no categorical eligibility. Every model that answered applied only the federal NSLP test (130% of FPL, or SNAP/TANF categorical eligibility) and returned 0. None of them let California's universal program override that test."
us,scenario_099,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_099,payroll_tax,44,llm_error,state_local_rule,False,,"The federal part is simple: 7.65% FICA on $165,000 of gross wages is $12,622.50 ($10,230 Social Security plus $2,392.50 Medicare), and there is no Additional Medicare Tax. The answers split on California's mandatory employee SDI. For 2026 it is 1.3% of all wages with no cap, which is $2,145 on $165,000. Most models either left SDI out or used last year's 1.2% ($1,980) or an older 1.1% ($1,815). A smaller group also wrongly took the employer-sponsored insurance premiums out of the FICA/SDI wage base, even though the engine taxes the full gross wages."
+us,scenario_099,payroll_tax,45,llm_error,state_local_rule,False,,"Employee FICA is $12,622.50: 7.65% of the full $165,000 in gross wages. Traditional 401(k) deferrals and the listed ESI premium amounts do not reduce this base, and there is no Additional Medicare Tax. California's mandatory employee SDI adds 1.3% of all wages with no cap in 2026, which is $2,145, so the total is $14,767.50. Most models either left SDI out or used the older 1.1%/1.2% rates. A smaller group shrank the wage base by subtracting ESI premiums or 401(k) deferrals."
us,scenario_099,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -778,7 +778,7 @@ us,scenario_099,spouse_medicaid_eligible,1,parse_contract_failure,missing_output
us,scenario_099,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_099,state_income_tax_before_refundable_credits,49,llm_error,taxable_income_or_deductions,False,,"The correct path starts from CA AGI of $160,066.80: wages net of $3,088 in 401(k) deferrals, a $3,000 capital-loss cap, the $337.50 educator expense added back, and no IRA deduction. California itemized deductions are only $8,653.50 of real estate tax plus $12,621 of charity, for $21,274.50. Medical costs fall below the 7.5% floor, unreimbursed employee expenses fall below the 2% floor, and employer insurance premiums do not reduce wages. Itemizing beats the $11,412 standard deduction and leaves $138,792.30 taxable, which carries $5,871.48 of bracket tax. The full $1,256 of personal and dependent exemption credits then applies. Most models went wrong in one of four ways: adding a medical deduction or subtracting ESI premiums, using the standard deduction instead of itemizing, misestimating 2026 bracket tax, or getting the exemption credits wrong."
+us,scenario_099,state_income_tax_before_refundable_credits,50,llm_error,taxable_income_or_deductions,False,,"The correct path has four steps. First, CA AGI is $160,066.80: federal AGI with the $337.50 educator deduction added back, after the $3,088 of 401(k) deferrals and the $3,000 capital-loss cap, and with no IRA deduction. Second, the couple itemizes $21,274.50 ($8,654 real estate tax plus $12,621 charity). There is no medical deduction because qualifying expenses fall below the 7.5% floor, and the employee business expenses fall below the 2% floor. Third, MFJ bracket tax on $138,792.30 is $5,871.48. Fourth, subtracting the $1,256 of exemption credits (2×$153 personal plus 2×$475 dependent) leaves $4,615.48. Wrong answers fall into a few groups: models that invented a medical deduction (double-counting premiums or including OTC costs), models that subtracted the employer-paid ESI premiums from income, models that took the standard deduction instead of itemizing, and models that misstated the bracket tax or the exemption-credit amounts (often dropping them entirely)."
us,scenario_099,state_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_099,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_100,child1_chip_eligible,6,llm_error,categorical_eligibility,False,,"CHIP eligibility requires the child not to qualify for Medicaid. At age 6 and this income, Child 1 qualifies for Montana Medicaid under the OLDER_CHILD category, so Medicaid eligibility categorically precludes CHIP eligibility; the CHIP income ceiling and existing employer-sponsored insurance do not alter that result."
@@ -798,11 +798,11 @@ us,scenario_100,head_wic_eligible,4,llm_error,categorical_eligibility,False,,"WI
us,scenario_100,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_100,payroll_tax,7,llm_error,payroll_tax_base,False,,"The trap is the payroll-tax wage base: PolicyEngine assesses employee OASDI at 6.2% and HI at 1.45% on the full $5,915 of employment income, giving $366.71 + $85.76 = $452.48. None of the listed health inputs touch that base — the $20,644 employer-sponsored insurance premium and the two $564 individually paid premiums are health-coverage and medical-expense inputs, not Section 125 salary reductions — and the $710 FLSA overtime premium is already inside the annual gross wage figure rather than additional to it. Traditional 401(k) and IRA amounts also leave FICA wages untouched, and Montana imposes no mandatory employee-side state payroll tax, so the answer is exactly 7.65% of wages. The wrong answers split into base errors (subtracting premiums to $4,787 or to $0, or adding the overtime premium to reach $6,625) and, for the three models that found the right base, an arithmetic slip, an invented rounding, and a value/explanation mismatch."
us,scenario_100,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price eligibility is mutually exclusive with the superior free-meals tier. This household is at 44% of the federal poverty guideline and also meets categorical eligibility, so its school_meal_tier is FREE and reduced_price_school_meals_eligible is 0."
-us,scenario_100,snap,51,prompt_ambiguity,age_disability,False,,"The reference counts about $505/month of Montana TANF cash as unearned income, so SNAP gross income is $998.31 rather than just the $492.92 of wages. It also applies a $453.64 excess shelter deduction based on the utility allowance, even though no rent is listed. It gives no excess medical deduction, because the head receives no SSI or other disability-based benefit and so does not count as a SNAP elderly/disabled member. That leaves net income of $237.09 and a $71.10 household contribution. The benefit is $785 − $71.10 = $713.90/month for Jan–Sep and $733.60/month for Oct–Dec under the higher FY2027 allotment of $802.60. Most models left out TANF income and added a medical deduction, which pushed them toward the maximum allotment. Others included TANF and took no shelter deduction, as policyengine-us 2.60.0 does, or zeroed out the benefit without a valid reason. Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, since fixed upstream by PolicyEngine/policyengine-us#10033, which the policyengine-us 2.41.0 references include) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification."
+us,scenario_100,snap,52,prompt_ambiguity,age_disability,False,,"The engine counts Montana TANF cash (about $505.39/month) as unearned income on top of $492.92 in monthly earnings, so gross income is $998.31. It then subtracts the $209 standard deduction, the $98.58 earned-income deduction and a $453.64 excess shelter deduction. It gives no excess medical deduction, because the head receives no SSI or other disability benefit and so does not count as a SNAP-disabled member. That leaves net income of $237.09 and a $71.10 contribution, subtracted from $785 in January–September and the higher FY2027 allotment in October–December, which totals $8,625.89. Most models left out the TANF income and the shelter deduction and instead took a medical deduction for the disabled head. That pushed them close to the maximum allotment (about $9,000–$9,420). The few models that did count TANF still missed the shelter deduction and came in too low (about $6,900–$7,800). Developer adjudication (2026-09-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (age_disability). The output is removed from scoring for every model: its reference depends on an engine input the certified household data never carried and the prompt therefore never listed, and a careful reader could take the stated facts the other way. Recomputed with policyengine-us 1.755.4 (the version that produced the references) the reference moves from 8625.889648 to 8917.490234 under the alternative reading. Neither reading is established by the facts; rows that matched the frozen value leave the score along with rows that did not. Judge diagnoses are retained as description; the class prompt_ambiguity records that the reference, not the model, is indeterminate here. SSI stays $0 under either reading ($2,800 in the bank exceeds the $2,000 resource limit); SNAP moves through the elderly-or-disabled SNAP rules. The 2026-09-22 audit recomputes the alternative as 8,844.00 with every publication convention and upstream fix (the frozen-engine value above was 8,917.49). r30_snap_heat_and_eat_sua (engine defect, since fixed upstream by PolicyEngine/policyengine-us#10033, which the policyengine-us 2.41.0 references include) also moves this output on the stated facts, from 8,556.00 to 6,924.00; the record keeps its 2026-09-05 classification."
us,scenario_100,ssi,9,llm_error,categorical_eligibility,False,,"The generic `is disabled` household fact does not make the head categorically eligible for SSI under PolicyEngine; the controlling SSI-specific indicator is `is_ssi_aged_blind_disabled=False`. All three individuals fail the aged-65-or-older, blind, or qualifying-disabled requirement, so income exclusions, resources, and benefit-rate calculations are never reached and household SSI is $0."
-us,scenario_100,state_refundable_credits,49,llm_error,state_local_rule,False,,"For 2026 the Montana EITC is refundable and equals 20% of the federal EITC. It is the only refundable Montana credit for this household. The federal EITC uses earned income included in gross income, so the $3,859 of traditional 401(k) deferrals comes out of the $5,914.72 in wages, leaving $2,055.72. At the 40% two-child phase-in rate, that gives a federal EITC of $822.29, and 20% of it is $164.46. The models split into three groups: those that said Montana has no refundable EITC (or no income tax), those that used the old 3% or 10% rate, and those that ran the EITC on gross wages ($2,366 federal) instead of wages net of 401(k) deferrals. Some models made more than one of these errors, and some also added a $1,200 young-child credit that is not part of this household's refundable credits."
-us,scenario_100,tanf,51,llm_error,thresholds_rates,False,,"The reference uses a 3-person assistance unit. Monthly wages are $5,915/12 = $492.92. After Montana's $200 plus 25%-of-remainder earned-income disregard, countable income is $219.69 a month. That amount is subtracted from Montana's 2026 three-person payment standard of about $725.10, which gives about $505.41 a month, or $6,064.96 a year. The $2,800 bank balance is under the $3,000 resource limit, the vehicle is excluded, and the head gets no SSI because $2,800 is over SSI's $2,000 resource limit. Most wrong answers used an outdated $588 (or other too-low) payment standard, which gives about $4,420, or a 50% disregard. Others wrongly ruled the family ineligible: they assumed an asset limit of $1,000 or $2,000, counted SSI income the head does not receive, said the program was not modeled or not taken up, or added the overtime premium on top of wages that already include it."
-us,scenario_101,federal_income_tax_before_refundable_credits,43,llm_error,thresholds_rates,False,,"The correct path runs from AGI of $101,130 ($103,000 wages minus the fully deductible $1,870 capital loss; the tax-exempt pension and veterans benefits are excluded). Subtract the 2026 OBBBA single standard deduction of $16,100 to get taxable income of $85,030. Then apply the 2026 brackets (10% to $12,400, 12% to $50,400, 22% above): $1,240 + $4,560 + $7,618.60 = $13,418.60. Three mistakes account for most wrong answers. Some models assumed the TCJA sunset (personal exemption plus 15%/25% brackets), but OBBBA made the TCJA rates permanent. Some used stale or guessed standard deductions and bracket thresholds. Others took the OBBBA qualified vehicle-loan interest deduction, but the household facts never establish a qualifying new, U.S.-assembled vehicle loan, so the $1,400 of auto loan interest is not deductible."
+us,scenario_100,state_refundable_credits,50,llm_error,state_local_rule,False,"The 2026 trace shows only the Montana EITC, but Montana's refundable child tax credit under HB 268 (2023) pays $1,200 for each child under 6. Child 2 is 5, the AGI of about $2,056 is far below the $56,000 limit, and the household has earned income, so PolicyEngine's mt_ctc appears to be missing from mt_refundable_credits or wrongly zeroed.","The reference amount is entirely the Montana EITC. For 2026 it is refundable and equals 20% of the federal EITC, after HB 337 (2025) raised it from the 10% rate used in 2024–2025 and the 3% rate used before that. The federal EITC is based on earned income net of traditional 401(k) deferrals: $5,914.72 − $3,859 = $2,055.72. At the 40% phase-in rate for two children, that gives $822.29, and 20% of it is $164.46. Most models either applied a stale 10% or 3% rate, applied 40% to gross wages ($2,366), or wrongly said Montana has no refundable EITC. Developer adjudication (2026-11-11): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). The judge flagged the reference on the hypothesis that Montana's HB 268 (2023) pays a refundable $1,200 credit for each child under 6. HB 268 never became law: the House Appropriations Committee tabled it on February 23, 2023, and it died there. No later act created a Montana child tax credit; Title 15, chapter 30, part 23 of the 2025 Montana Code has none, and the Department of Revenue's 2025 Form 2 instructions name only HB 129 and HB 337 among the changes for tax year 2026. For 2026 the household's one Montana refundable credit is the earned income tax credit, 20% of the federal credit: 40% of $2,055.72 (wages of $5,914.72 less $3,859 of traditional 401(k) deferrals) gives $822.29, and 20% of that gives $164.46, the reference. The rows are model errors. Reference affirmed (Mont. Code Ann. 15-30-2318(2) and (4) (2025), as amended by Ch. 227, Laws of 2025 (HB 337), secs. 3 and 6(1): a refundable credit of 20% of the federal earned income credit for the tax year beginning January 1, 2026; Mont. Code Ann. Title 15, ch. 30, part 23 (2025), which holds no child tax credit; minutes of the Montana House Appropriations Committee, February 23, 2023, which tabled HB 268)."
+us,scenario_100,tanf,52,llm_error,thresholds_rates,False,,"The engine puts the head and both children in a 3-person Montana TANF unit. No SSI income is counted. The $2,800 in bank assets is under the $3,000 resource limit, and the vehicle is excluded. Countable earnings are ($5,915/12 − $200) × 75% = $219.69/month, and the 2026 three-person payment standard is about $725.10/month, so the grant is about $505.41/month, or $6,064.96 a year. Most wrong answers either used an old or made-up payment standard (often the outdated $588, or a 50% disregard) or wrongly gave $0. The $0 answers came from made-up asset limits, counting the vehicle, assuming SSI receipt, or assuming the program isn't modeled."
+us,scenario_101,federal_income_tax_before_refundable_credits,44,llm_error,thresholds_rates,False,,"The reference is AGI of $101,130 (wages minus the $1,870 capital loss), less only the OBBBA 2026 single standard deduction of $16,100, giving taxable income of $85,030. That income is taxed at 10% up to $12,400, 12% up to $50,400 and 22% above, for $1,240 + $4,560 + $7,618.60 = $13,418.60, with no nonrefundable credits. PolicyEngine allows no deduction for the listed $1,400 of auto loan interest. The facts never show a qualifying new, U.S.-assembled vehicle, so models that took a $1,000–$1,400 auto-loan deduction came out $220–$308 low. Most other misses came from stale or guessed parameters: 2025 or projected standard deductions and bracket thresholds, or a TCJA sunset with personal exemptions and 15%/25% brackets. A few models did the math correctly and then submitted a different number. Developer adjudication (2026-11-11): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The judge labeled eight models' rows reference_later_law (deepseek-v4-pro-0813, gemini-3-flash-preview, gemini-3.5-flash, gemini-3.6-flash, gemini-3.7-flash, gpt-5.5, grok-4.5 and grok-build-0.1) because they applied the pre-TCJA rules scheduled to return in 2026 (a smaller standard deduction, a personal exemption and a 15% second bracket). The reference applies the permanent TCJA parameters of P.L. 119-21, enacted July 4, 2025, before the July 3, 2026 reference freeze, at the 2026 single amounts of Rev. Proc. 2025-32, released October 9, 2025 (a $16,100 standard deduction, with the 12% rate starting at $12,400 and the 22% rate at $50,400), and the benchmark scores law published before the freeze; the rows are model errors, as the 2026-09-29 and 2026-10-14 waves decided for the same rule in scenario_007's and scenario_009's federal income tax."
us,scenario_101,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_101,payroll_tax,8,llm_error,other,False,,"The correct calculation is simple. Employee Social Security is 6.2% of the full $103,000 of wages, or $6,386, because the wages are below the 2026 wage base. Employee Medicare is 1.45%, or $1,493.50. No Additional Medicare Tax applies below $200,000, and Texas has no employee state payroll tax, so the total is $7,879.50. Most models reached exactly $7,879.50 in their reasoning and then submitted a different number with a made-up ""adjustment."" One model treated the listed health insurance premiums as a pre-tax payroll exclusion, another submitted zero, and one gave no output."
us,scenario_101,state_income_tax_before_refundable_credits,1,llm_error,state_local_rule,False,,"Texas imposes no individual state income tax, so a Texas resident has $0 of state income tax before refundable credits regardless of the listed taxable-income deductions or credits. Taxes from other jurisdictions do not apply because the household resides in Texas."
@@ -837,7 +837,7 @@ us,scenario_108,reduced_price_school_meals_eligible,1,parse_contract_failure,mis
us,scenario_108,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_108,snap,48,llm_error,categorical_eligibility,False,,"Wisconsin FoodShare uses broad-based categorical eligibility through TANF non-cash benefits. Its gross limit is 200% FPL, or $2,608.33 a month for one person, and the household's $2,581.83 falls below it. That makes the household categorically eligible and waives the 100% FPL net income test. Because 30% of the $2,036 net income ($611) exceeds the $298 maximum allotment, this one-person household gets the FY2026 minimum allotment of $24 (8% of $298, rounded to the nearest dollar), or $288 a year. Most models applied the net income test or the ""contribution exceeds max = $0"" shortcut and missed both the categorical-eligibility pathway and the minimum benefit. Two models found the right pathway but used the older $23 minimum, and a few submitted large benefit amounts that their reasoning did not support."
us,scenario_108,state_income_tax_before_refundable_credits,12,llm_error,state_local_rule,False,,"Every model that gave an answer counted the $10,392 non-SS survivor benefits as taxable, which produced a Wisconsin AGI of $18,048. The $20,590 of annual income in the engine's derivation is wages plus Social Security survivor benefits. The engine treats the $10,392 survivor benefits as nontaxable, so they enter neither federal nor Wisconsin AGI. Social Security is exempt in Wisconsin, so the only taxable income left is $7,656 of wages, and the Wisconsin standard deduction of about $13,000+ brings taxable income to $0. Separately, even on the models' own taxable-income figures, Wisconsin's nonrefundable school property tax credit for renters would have wiped out their liabilities: $7,200 rent becomes $1,440 of rent-equivalent property tax, and 12% of that is a $172.80 credit, more than any model's pre-credit tax of $94 to $162. No model got both points right, so none reached $0."
-us,scenario_108,state_refundable_credits,51,prompt_ambiguity,state_local_rule,False,"PolicyEngine leaves the listed $10,392 generic `survivor_benefits` out of both AGI and wi_homestead_income. Wis. Stat. § 71.52(5) adds the gross amount of pensions and annuities, including survivor annuities, to Wisconsin AGI when computing homestead household income. Counting it gives $30,982, which is above the $24,680 limit, so the credit would be $0.","The deciding step is what goes into Wisconsin homestead household income. PolicyEngine's wi_homestead_income counts only wages ($7,656) and Social Security survivor benefits ($12,934), for $20,590, and leaves out the generic $10,392 survivor benefits. That keeps income under the $24,680 limit, and with property tax set at 20% of the $7,200 rent ($1,440) the credit is 0.8 × (1,440 − 0.08785 × (20,590 − 8,060)) = $271.39. Most models added the $10,392 survivor benefits, reached $30,982, and set the credit to zero. The rest either skipped the homestead credit entirely, left out Social Security, or used 25% of rent and the wrong phase-out parameters. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 271.39; the alternative is 0.00. The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out. Reference depends on an unlisted input; output excluded from scoring (source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits)."
+us,scenario_108,state_refundable_credits,52,prompt_ambiguity,state_local_rule,False,,"The whole $271.39 is the refundable Wisconsin homestead credit. In PolicyEngine, homestead household income (wi_homestead_income) is wages of $7,656 plus Social Security of $12,934, which is $20,590. The separately listed $10,392 of survivor benefits is not counted, so the household is under the $24,680 limit. Rent is converted to property tax at 20% ($7,200 → $1,440), and the credit is 80% × (1,440 − 8.785% × (20,590 − 8,060)) = $271.39. Most models added the $10,392 survivor benefits to reach $30,982, decided the household was over the limit, and answered $0. Others ignored the homestead credit or wrongly said it needs tax liability. The near-misses used the 25% rent conversion capped at $1,460, or the wrong rate and threshold. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 271.39; the alternative is 0.00. The prompt lists survivor benefits other than Social Security without saying what they are. Wisconsin homestead household income counts the gross amount of a survivor pension or annuity (Wis. Stat. 71.52(6); Schedule H line 9d), which is the reading under the alternative value; the reference leaves them out. Reference depends on an unlisted input; output excluded from scoring (source of survivor_benefits (pension or annuity, or another survivor payment) and taxability of survivor_benefits)."
us,scenario_109,child1_chip_eligible,6,llm_error,health_coverage,False,,"CHIP is limited to children who do not qualify for Medicaid. At approximately $30,000 of household income, the age-9 child qualifies for Florida Medicaid in the OLDER_CHILD category, so Medicaid eligibility precludes CHIP eligibility even though the income is below CHIP's upper limit."
us,scenario_109,child1_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Florida's refusal of adult Medicaid expansion does not touch children's coverage: the state's mandatory MAGI pathway for the older-child category (ages 6-18) runs to 133% FPL, and a 9-year-old in a household of five with roughly $21,600 of MAGI sits at 0.72 FPL — barely half the threshold. The only computation required is household MAGI from the $30,000 of self-employment income divided by the family-of-five poverty guideline, then compared with 133% FPL; no asset test, no enrollment fact, and no SSI linkage enters the children's MAGI pathway. Both models returned 0 without ever running that ratio, treating Medicaid eligibility as something the prompt had to assert rather than something the stated income and age determine."
us,scenario_109,child2_chip_eligible,6,llm_error,health_coverage,False,,"CHIP eligibility requires the child to be ineligible for Medicaid, not merely under age 19 and below CHIP's upper income limit. At age 6 and this household income, child2 qualifies for Medicaid under Florida's OLDER_CHILD category, so the Medicaid–CHIP mutual-exclusivity rule makes child2 ineligible for CHIP."
@@ -856,14 +856,14 @@ us,scenario_109,head_medicaid_eligible,1,llm_error,categorical_eligibility,False
us,scenario_109,head_wic_eligible,2,llm_error,categorical_eligibility,False,,"WIC categorical eligibility is individual, not household-wide: only a pregnant, postpartum, or breastfeeding woman, an infant, or a child under age five can qualify. The infant satisfies that requirement, but the 30-year-old Head has no listed pregnancy, postpartum, or breastfeeding status, so the Head remains ineligible despite the household meeting the income test."
us,scenario_109,reduced_price_school_meals_eligible,2,llm_error,categorical_eligibility,False,,"Reduced-price and free school meals are mutually exclusive output tiers in this calculation. The household’s 0.78 school-meal FPG ratio and categorical eligibility place it in the superior FREE tier, so it receives positive free-meal support and no reduced-price support."
us,scenario_109,self_employment_tax,6,llm_error,other,False,,"The Schedule SE computation here is fully determined: net earnings from self-employment are 92.35% of $30,000 = $27,705, entirely below the Social Security wage base, so the tax is $27,705 × 15.3% = $4,238.865, which PolicyEngine reports as $4,238.86. Every model named the correct base fraction, the correct 15.3% combined rate, and correctly noted the wage base is not binding — none missed a rule. The separation is purely in executing the product: each model reported a whole-dollar figure that is $2 to $6 away from $4,238.87, whereas correct rounding of the true product to whole dollars gives $4,239. The trap is arithmetic precision on 0.9235 × 0.153 × $30,000, not any element of the SE tax rule."
-us,scenario_109,snap,45,llm_error,thresholds_rates,False,,"The benefit follows directly from FY2026 parameters: monthly gross income is $2,500. Subtracting the $500 earned income deduction (20%) and the $261 standard deduction for five people leaves $1,739. The $800 rent is below half of that ($869.50), and no utility costs are listed, so there is no excess shelter deduction and no SUA. The expected contribution is 30% of $1,739, rounded up to $522, and $1,183 − $522 = $661 per month, or $7,932 per year. Most models missed by using stale or guessed standard deductions and maximum allotments (FY2025's $1,158 and $254, or the 6+ person $299), by inventing a shelter or utility deduction, or by subtracting SE tax. The closest answers reached $661.30 per month because they skipped the rule that rounds the 30% contribution up to a whole dollar. A few wrongly found the household ineligible or miscounted its size. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 8,020.55 is regenerated as 7,932.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze): If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment. Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 CFR 273.10(e)(2)(ii)(A))."
+us,scenario_109,snap,46,llm_error,thresholds_rates,False,,"The calculation is $2,500 in monthly self-employment earnings, minus the 20% earned income deduction ($500) and the FY2026 standard deduction for five people ($261), for net income of $1,739. Rent of $800 is below half of that amount and no utility costs are listed, so there is no excess shelter deduction. The 30% contribution of $521.70 rounds up to $522, and $1,183 minus $522 is $661 a month, or $7,932 a year. Models that got the formula right missed by rounding: they kept $661.30 a month and reported $7,935.60 or $7,936. Most other models used FY2025 or made-up parameters, such as a $254 or $236 standard deduction or a $1,155–$1,220 maximum. A few added a shelter deduction or utility allowance that does not apply, or wrongly denied eligibility. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (thresholds_rates). The frozen 8,020.55 is regenerated as 7,932.00: it applies the publication rule: SNAP October-December 2026 hold the FY2026 schedule (the SNAP uprating index behind the maximum allotments, deductions, shelter cap and utility allowances), the last USDA published before the 2026-07-03 reference freeze; USDA published FY2027 on 2026-08-21. The poverty guideline is the 2026 HHS guideline, published in January 2026. The engine's SNAP rounding defects are root causes r26, r27, r28 and r31, fixed upstream and applied with the convention.; it corrects an engine defect fixed upstream (fixed in PolicyEngine/policyengine-us#9318 (merged 2026-08-25), after the reference freeze): If 30% of net income ends in cents, the state rounds it up to the next dollar or rounds the allotment down to the next dollar; with a whole-dollar maximum allotment both give the same whole-dollar allotment. Reference regenerated (7 U.S.C. 2012(u), 2017(a); USDA FY2026 SNAP COLA memorandum (signed 2025-08-14); 7 CFR 273.10(e)(2)(ii)(A))."
us,scenario_109,spouse_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Florida’s non-expansion Medicaid program does not provide a general low-income pathway for a nondisabled 29-year-old adult. The spouse qualifies through no categorical pathway: household MAGI exceeds Florida’s parent/caretaker limit, and the presence of an infant does not establish pregnancy or postpartum status."
us,scenario_109,spouse_wic_eligible,8,llm_error,categorical_eligibility,False,,"WIC eligibility for an adult requires the adult herself to be pregnant, breastfeeding, or within the applicable postpartum period; merely living with an infant does not establish any of those statuses. The prompt makes every unlisted status false, so the spouse is not pregnant, postpartum, or breastfeeding and fails categorical eligibility regardless of household income."
-us,scenario_110,federal_income_tax_before_refundable_credits,49,llm_error,taxable_income_or_deductions,False,,"The 2026 rules are permanent TCJA as extended by OBBBA. That means a $16,100 single standard deduction and brackets of 10% to $12,400, 12% to $50,400, 22% to $105,700 and 24% above that. The traditional IRA deduction is phased out for an active plan participant with AGI near $170k. ESI premiums are not taken out of the $95,755 of wages. No overtime income is listed, so there is no overtime deduction. Most often missed: the new non-itemizer cash charitable deduction ($387, up to $1,000 for a single filer). It cuts taxable income to $153,455, so ordinary tax is $19,615 on $112,572 and the 15% tax on $40,884 of qualified dividends and long-term gains is $6,133, for $25,748. Many models stopped at $25,841 because they left out the $387 deduction. Others brought back pre-TCJA personal exemptions and miscellaneous itemized deductions, subtracted ESI premiums from wages, or made up an overtime deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,50,llm_error,taxable_income_or_deductions,False,,"Under OBBBA's permanent TCJA rules for 2026, AGI is $169,943. That is wages less only the $4,245 traditional 401(k), with no deduction for the traditional IRA (phased out for an active plan participant) and no exclusion for the ESI premium input. Taxable income takes off the $16,100 single standard deduction plus the new non-itemizer cash charitable deduction ($387; non-cash gifts don't count), leaving $153,455. The $112,572 of ordinary income is taxed under the 2026 brackets ($12,400/$50,400/$105,700) for $19,615, and $40,884 of qualified dividends and long-term gains is taxed at 15% for $6,133. The largest group of models landed on $25,841 because they left out the $387 charity deduction. Others went wrong by inventing an overtime deduction with no qualified overtime input, excluding ESI premiums from wages, taking the IRA deduction, using sunset-era personal exemptions and miscellaneous itemized deductions, or dropping the capital gains tax."
us,scenario_110,federal_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_110,payroll_tax,17,llm_error,payroll_tax_base,False,,"The reference applies employee FICA to the full $100,000 of stated gross wages: 6.2% Social Security ($6,200, well under the wage base) plus 1.45% Medicare ($1,450). That totals $7,650, and there is no Additional Medicare Tax because wages are below $200,000. The household facts never say the $6,889 employer-sponsored insurance premium is a Section 125 salary reduction, and the prompt says not to infer unlisted facts. The largest group of models still subtracted it from FICA wages and reported $7,123. The other wrong answers came from excluding the 401(k) deferral from FICA wages, applying Additional Medicare Tax to non-wage income, or submitting a value that did not match the model's own reasoning."
us,scenario_110,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_110,state_income_tax_before_refundable_credits,51,llm_error,thresholds_rates,False,,"Ohio AGI equals federal AGI of $169,942.94: wages net of the $4,245 traditional 401(k), plus interest, dividends, capital gains and estate income. There is no IRA deduction, no Ohio addback, and no itemized or standard deduction. Subtracting the $1,900 personal exemption for income over $80,000 leaves $168,042.94 of taxable income. Under the 2026 Ohio schedule, the tax is a fixed $332 base plus 2.75% of the amount over $26,050, which gives $4,236.81, and no nonrefundable credits apply. The most common mistakes were dropping the $332 base (which gives $3,905), using the repealed 3.5%/3.125% bracket above $100,000 or the old $360.69 base, and misstating AGI, mostly by subtracting the $6,889 of employer health premiums or the phased-out $198 IRA contribution."
+us,scenario_110,state_income_tax_before_refundable_credits,52,llm_error,thresholds_rates,False,,"Ohio AGI is $169,942.94. That is federal AGI with the $4,245 401(k) deferral excluded from wages, no deduction for the $198 IRA contribution (it is fully phased out for a 401(k) participant at this income), and no subtraction for employer insurance premiums. The $1,900 personal exemption for OAGI above $80,000 leaves taxable income of $168,042.94. The 2026 Ohio schedule is $332 plus 2.75% of taxable income above $26,050, which gives $4,236.81 with no nonrefundable credits. The most common miss was leaving out the $332 base, which produced about $3,905. Other models used the old 3.5% bracket above $100,000, the old $360.69 base, the wrong exemption amount, or shrank AGI by deducting the IRA contribution or the $6,889 employer insurance premiums."
us,scenario_110,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_111,federal_income_tax_before_refundable_credits,2,llm_error,taxable_income_or_deductions,False,,"The IRA distribution produces $18,240 of adjusted gross income, while the Social Security retirement benefit is not taxable under the joint-return provisional-income thresholds. The applicable standard deduction eliminates taxable income, so the tax computation yields $0; both models expressly derived zero taxable income and zero liability but then submitted unsupported positive amounts."
us,scenario_111,head_medicaid_eligible,4,llm_error,categorical_eligibility,False,,"At 72, the head is outside the ACA expansion-adult category, which covers ages 19–64 up to 138% FPL. Their MAGI is also about 202% FPL, so that pathway fails twice. The head gets no SSI, so the SSI-recipient route is closed. The aged/blind/disabled route has an income limit near the SSI/100% FPL level, roughly $1,000–$1,300 a month. The head's unearned income is about $3,637 a month, so it is far over that limit even after the $20 general disregard. All four models treated being aged, blind or disabled plus low assets as enough to qualify and skipped the income test, which leaves medicaid_category = NONE."
@@ -877,17 +877,17 @@ us,scenario_112,head_medicaid_eligible,5,llm_error,categorical_eligibility,False
us,scenario_112,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC requires both financial eligibility and membership in a qualifying demographic category: pregnant, postpartum, or breastfeeding women, infants, or children under age five. The 31-year-old head has no listed qualifying status, and unlisted statuses are false, so passing the income test does not establish WIC eligibility."
us,scenario_112,payroll_tax,7,llm_error,payroll_tax_base,False,,"Employer-sponsored insurance premiums are not automatically employee pre-tax cafeteria-plan deductions from FICA wages; the listed $5,923 remains subject to employee Social Security and Medicare taxes. PolicyEngine computes $367.22 of Social Security tax and $85.88 of Medicare tax, totaling $453.10, with no Additional Medicare Tax or mandatory Texas employee payroll tax."
us,scenario_112,self_employment_tax,4,llm_error,payroll_tax_base,False,,"Self-employment tax applies only to positive net earnings from self-employment, after determining the applicable self-employment tax base. The household’s relevant net income is approximately -$2,101, so there are no positive net self-employment earnings and the tax base is zero; treating the $1,920 farm rent entry as a standalone positive self-employment tax base produces the wrong liability."
-us,scenario_112,snap,51,prompt_ambiguity,categorical_eligibility,False,"PolicyEngine's SNAP unearned-income source list leaves out farm_rent_income, so the $1,920 of passive farm rent never enters SNAP gross income. 7 CFR 273.9(b)(2)(ii) counts rental-property income as unearned income when the member does not actively manage the property 20+ hours a week, and counting it raises net income to about $1,429/month, above the 100% FPL net limit of $1,304.17, which makes SNAP $0.","The $287.68 annual reference is frozen and excluded from scoring because weekly work hours were unlisted. The 2.60.0 recheck returns $0 under its default reading, which does not settle the ambiguity. The current engine counts farm rent as SNAP unearned income, and Texas BBCE does not apply the net income test to this household. For an eligible one-person household, the held FY2026 schedule gives a rounded $24 monthly minimum allotment. Most models counted the farm rent, which put net income near $1,430, above 100% FPL, so they found the household ineligible. Others either dropped the $13,000 of financial assistance and got large benefits, or treated '30% of net income exceeds the maximum' as $0 and skipped the minimum allotment. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_engine_defect; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 287.68; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr)."
-us,scenario_114,federal_income_tax_before_refundable_credits,51,prompt_ambiguity,taxable_income_or_deductions,False,,"Most models got AGI right ($106,573). Some applied pre-TCJA ""sunset"" law or omitted the OBBBA senior deduction; others used different readings of unlisted withholding and Medicare inputs. The frozen reference uses the 2026 OBBBA rules: the 10%/12%/22% brackets with breakpoints at $12,400 and $50,400, and the $6,000 senior deduction phased down by 6% of MAGI above $75,000, leaving $4,105.65. It adds a 20% QBI deduction of $4,872.35 and itemized deductions of $24,787.23. The itemized total includes an unlisted $5,367.30 Virginia withholding estimate and an imputed $2,434.80 Medicare Part B premium. It excludes OTC costs and applies the 0.5%-of-AGI charity floor, leaving the reference's taxable income at $72,807.33 and tax at $10,729.61. Recorded alternatives are $11,969.80 without withholding, the local sales-tax estimate or a Part B premium, and $11,642.16 using Virginia liability without a Part B premium; several models match these alternatives. This output remains excluded from scoring for dependence on unlisted inputs. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 10,729.61; the alternative is 11,137.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Virginia withholding on federal AGI ($5,367.30), while read as the household paying its 2026 Virginia income tax during the year the deduction takes the engine's liability ($3,514.15). The reference's medical deduction also counts a modeled Medicare Part B premium ($2,434.80) the prompt does not list; without it the output is 11,265.27. Found in the 2026-10-05 audits of the state income tax in SALT (reference_audit/2026-10-05) and of the Part B premium (reference_audit/2026-10-05-medicare-part-b). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set; also Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)."
+us,scenario_112,snap,52,prompt_ambiguity,categorical_eligibility,False,"PolicyEngine leaves farm_rent_income out of SNAP countable income: gross income of $1,576.90 equals wages ($493.58) plus financial assistance ($1,083.33), with no farm rent. Under 7 CFR 273.9(b), rental income counts and no 273.9(c) exclusion covers it, and adding the $160/month raises net income to about $1,429, above the $1,304.17 net-income limit the derivation applies.","PolicyEngine's SNAP gross income counts only wages ($493.58/mo) and financial assistance ($1,083.33/mo), for $1,576.90. That is 121% of FPL. It does not count the $160/mo of farm rent. After the 20% earned-income deduction and the ~$209 standard deduction, net income is $1,269.19 (97% of FPL), so the one-person household passes the gross test, the net test and Texas BBCE through TANF non-cash assistance. Because 30% of net income ($380.70) exceeds the $298 maximum, the household gets the one/two-person minimum allotment of $23.84/mo ($287.68 a year). Most models counted the farm rent, which pushes gross income above 130% of FPL or net income above 100%; others applied the 30% formula without the minimum benefit, and a few left out the $13,000 of financial assistance and overstated the benefit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned reference_model_issue_fixed; adjudicated prompt_ambiguity (categorical_eligibility). The frozen reference is 287.68; the alternative is 0.00. The prompt lists no hours worked and says to treat unlisted numeric inputs as 0; the reference assumed 40 hours a week, which clears the SNAP work requirement for able-bodied adults without dependents. Under the zero-hours reading the time limit applies, which ends benefits after three countable months unless an exemption or area waiver applies; the alternative value is the engine's zero-hours result, not a certified entitlement. Reference depends on an unlisted input; output excluded from scoring (weekly_hours_worked_before_lsr)."
+us,scenario_114,federal_income_tax_before_refundable_credits,52,prompt_ambiguity,taxable_income_or_deductions,False,,"The 2026 return is filed under OBBBA, which kept the TCJA brackets and QBI deduction in place, removed the personal exemption and added the $6,000 senior deduction. That senior deduction falls to $4,105.65 at this MAGI and is allowed on top of itemized deductions. Itemizing wins at $24,787.23: charity of $10,913.78 after the 0.5%-of-AGI floor; SALT of $9,431.59, made up of real estate tax plus the engine's $5,367.30 estimate of Virginia income tax withheld; and medical expenses above 7.5% of AGI. The medical figure includes a $2,434.80 Medicare Part B premium and leaves out the $2,000 of over-the-counter costs. After the $4,872.35 QBI deduction, taxable income is $72,807.33, and 1,240 + 4,560 + 22% × 22,407.33 = $10,729.61. Wrong answers came from several mistakes: using pre-TCJA sunset law, dropping the senior deduction, taking the standard deduction instead of itemizing, leaving out or understating the Virginia income-tax part of SALT, counting OTC costs instead of the Part B premium, or making arithmetic and shortcut errors. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 10,729.61; the alternative is 11,137.31. The prompt lists no state income tax withheld or paid during 2026; policyengine-us 2.15.17 fills the state income tax part of the federal state and local tax deduction with state_withheld_income_tax, a formula estimate of Virginia withholding on federal AGI ($5,367.30), while read as the household paying its 2026 Virginia income tax during the year the deduction takes the engine's liability ($3,514.15). The reference's medical deduction also counts a modeled Medicare Part B premium ($2,434.80) the prompt does not list; without it the output is 11,265.27. Found in the 2026-10-05 audits of the state income tax in SALT (reference_audit/2026-10-05) and of the Part B premium (reference_audit/2026-10-05-medicare-part-b). Reference depends on an unlisted input; output excluded from scoring (state income tax withheld or paid during 2026, which the federal state and local tax deduction takes as its state income tax part; policyengine-us fills it with state_withheld_income_tax, a formula estimate on federal AGI that the household data never set; also Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)."
us,scenario_114,federal_refundable_credits,2,llm_error,credit_phaseout,False,,"EITC phaseout uses adjusted gross income or earned income, whichever is greater, rather than wages alone. This filer’s partnership/S-corp income, taxable retirement distributions, pension, taxable Social Security, and interest raise AGI far above the childless EITC limit, so every refundable credit component is zero."
us,scenario_114,payroll_tax,4,llm_error,payroll_tax_base,False,,"The requested payroll-tax output includes employee Social Security and employee Medicare taxes on the $22,700 of wages, while self-employment tax is reported separately and Virginia adds no mandatory employee payroll tax here. The stated components sum to $1,407.40 + $329.15 = $1,736.55; arbitrary adjustments, duplication, and invented state payroll taxes produce the wrong answers."
us,scenario_114,self_employment_tax,18,llm_error,payroll_tax_base,False,,"The household's only business-type input is $24,362 of partnership or S-corp income, and that amount is not net earnings from self-employment. S-corp pass-through income is never subject to SE tax. A partnership distributive share counts only for a general partner, and the prompt says any unlisted status is false, so no general-partner status can be assumed. With zero self-employment income, SE tax is $0. Every substantive wrong answer made the same mistake: each model counted the partnership/S-corp amount as SE earnings, then applied 15.3% to 92.35% of it, which gives about $3,442. Some models then added further arithmetic errors on top of that."
-us,scenario_114,state_income_tax_before_refundable_credits,51,prompt_ambiguity,taxable_income_or_deductions,False,,"Virginia AGI is $86,743.77, which is federal AGI minus the $19,828.80 of taxable Social Security. The single filer's $50,000 AFAGI threshold eliminates the $12,000 age deduction entirely. When the engine counts neither its unlisted Part B premium as a medical expense nor its $5,367.30 Virginia withholding estimate in federal SALT, it takes the standard deduction. As computed, the filer itemizes federally in the engine, so Virginia itemized deductions apply. Those are $19,419.94 in total. Charitable gifts are $10,913.78 after the 2026 0.5%-of-AGI floor. Medical is $4,441.86, which counts the $2,434.80 Medicare Part B premium but not the $2,000 of OTC expenses. Real estate tax is $4,064. Exemptions are $1,730 ($930 personal plus $800 aged), giving taxable income of $65,593.83 and tax of $3,514.15. The model answers include four recurring paths: taking a Virginia standard deduction, keeping an age deduction that is phased out, dropping the $800 aged exemption, or building itemized deductions as $17,518. That $17,518 figure omits the Part B premium and skips the charitable floor. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,514.15; the alternative is 3,654.15. The prompt lists no Medicare enrollment or Part B premium; policyengine-us 2.15.17 treats the 69-year-old head as enrolled (takes_up_medicare_if_eligible defaults to true) and adds the $2,434.80 standard Part B premium to medical expenses, which Virginia's itemized deductions carry from the federal ones; read with no premium paid, the output is the alternative. Found in the 2026-10-05 audit of the Part B premium (reference_audit/2026-10-05-medicare-part-b, scripts/sweep_part_b.py). Reference depends on an unlisted input; output excluded from scoring (Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)."
+us,scenario_114,state_income_tax_before_refundable_credits,52,prompt_ambiguity,taxable_income_or_deductions,False,,"Virginia AGI is $86,743.77, which is federal AGI minus the $19,828.80 of federally taxable Social Security. The $12,000 age deduction is fully phased out because AFAGI is more than $12,000 above the $50,000 single threshold. Taxable income is $65,593.83: Virginia AGI minus itemized deductions of $19,419.94 and exemptions of $1,730 ($930 personal plus $800 aged). The itemized figure counts charitable gifts after the 2026 floor of 0.5% of AGI and medical expenses that include the $2,434.80 Medicare Part B premium but exclude the $2,000 of over-the-counter costs. Most wrong answers made one or more of four mistakes: using the Virginia standard deduction instead of itemizing, applying the phased-out age deduction, leaving out the $800 aged exemption, or building itemized deductions without the charitable floor or the Part B premium. Developer adjudication (2026-10-05): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 3,514.15; the alternative is 3,654.15. The prompt lists no Medicare enrollment or Part B premium; policyengine-us 2.15.17 treats the 69-year-old head as enrolled (takes_up_medicare_if_eligible defaults to true) and adds the $2,434.80 standard Part B premium to medical expenses, which Virginia's itemized deductions carry from the federal ones; read with no premium paid, the output is the alternative. Found in the 2026-10-05 audit of the Part B premium (reference_audit/2026-10-05-medicare-part-b, scripts/sweep_part_b.py). Reference depends on an unlisted input; output excluded from scoring (Medicare enrollment and a Medicare Part B premium paid during 2026, which the medical expense deductions count; policyengine-us treats every Medicare-eligible person as enrolled (takes_up_medicare_if_eligible defaults to true, so medicare_enrolled follows is_medicare_eligible) and adds the modeled premium (medicare_part_b_premium) to medical expenses, and the household data never set either)."
us,scenario_114,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_115,head_medicaid_eligible,5,llm_error,categorical_eligibility,False,,"Age 77 and a reported disability do not by themselves place the head in an Alabama Medicaid eligibility category. PolicyEngine assigns medicaid_category = NONE; the 1.47 FPL MAGI amount therefore cannot establish eligibility through a MAGI category, and immigration eligibility alone is insufficient."
us,scenario_115,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_115,snap,5,llm_error,thresholds_rates,False,,"SNAP eligibility uses approximately $23,442 of annual household income here: $18,708 of Social Security plus $4,734 of tax-exempt interest. That income exceeds PolicyEngine’s applicable one-person SNAP eligibility limit, so the calculation stops at ineligibility and returns $0; deductions and minimum-allotment rules do not create a benefit for an ineligible household."
-us,scenario_115,state_income_tax_before_refundable_credits,50,llm_error,state_local_rule,False,,"Alabama exempts Social Security, but it does not follow the federal exclusion for municipal-bond interest. Alabama gross income includes interest on obligations of other states. PolicyEngine therefore puts the $4,734 of tax-exempt interest into Alabama AGI. Alabama allows a single filer only a $3,000 standard deduction at this AGI plus a $1,500 personal exemption, with no extra deduction for age or disability. That leaves $234 of taxable income, taxed at 2% for $4.68. Every model answered $0. Most treated federally tax-exempt interest as also exempt in Alabama. Several models considered adding it back but overstated the deductions as large enough to wipe out $4,734."
+us,scenario_115,state_income_tax_before_refundable_credits,51,llm_error,state_local_rule,False,,"Alabama does not follow the federal exclusion for municipal-bond interest. It taxes interest from obligations of other states, and PolicyEngine counts the full $4,734 of federally tax-exempt interest in Alabama AGI. Social Security is excluded from Alabama income. That leaves $4,734 of AGI, minus the $3,000 single standard deduction and the $1,500 personal exemption, which gives $234 of taxable income. At the 2% bracket the tax is $4.68. Most models assumed federally tax-exempt interest is also exempt in Alabama. The others added the interest back but overstated Alabama's deductions, using an age-65 bonus or a standard deduction of $3,500 or more that Alabama does not have, so they concluded the deductions wiped out the $4,734."
us,scenario_115,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_116,snap,1,llm_error,categorical_eligibility,False,,"SNAP’s minimum allotment applies only after a household passes the program’s eligibility tests; it cannot override failure of the gross-income test. At $52,800 of annual household income, this household exceeds the applicable 2026 gross-income limit, so the computation stops with no SNAP benefit."
us,scenario_116,spouse_medicaid_eligible,3,llm_error,categorical_eligibility,False,,"Florida did not expand Medicaid, so a childless, non-pregnant 42-year-old has no MAGI adult or parent category, even with MAGI at 0% FPL (veterans benefits are left out of MAGI). Disability alone does not create eligibility. The disabled pathway requires either receiving SSI or passing Florida's aged/disabled income test. The spouse's $52,800 in veterans benefits counts as unearned income under SSI-related rules, which pushes SSI to $0 and puts the spouse far above the aged/disabled income limit, so medicaid_category is NONE. All three models treated disability status as enough to qualify and never applied the income test that comes with the disabled pathway."
@@ -895,27 +895,27 @@ us,scenario_116,spouse_medicare_eligible,4,llm_error,age_disability,False,,"Unde
us,scenario_117,child3_early_head_start_eligible,5,llm_error,categorical_eligibility,False,,"Early Head Start eligibility requires both an age-qualified child and satisfaction of the program's low-income eligibility rules; being under age 3 is not independently sufficient. Although Child 3 meets the age condition, household income of approximately $250,733 is far above the applicable poverty-based income limit, so the income test produces no eligibility."
us,scenario_117,child3_head_start_eligible,1,llm_error,categorical_eligibility,False,,"Age alone does not establish Head Start eligibility; the child must also qualify through an applicable eligibility pathway, principally the program's low-income criteria in this case. This household's approximately $250,733 income exceeds the relevant poverty-based threshold, so Child 3 is not eligible."
us,scenario_117,child3_wic_eligible,1,llm_error,thresholds_rates,False,,"WIC child eligibility requires both categorical eligibility and financial eligibility; being age 2 satisfies only the child-category requirement. The household's approximately $250,733 annual income exceeds the WIC limit of 185% of the federal poverty guideline for a five-person household, so Child 3 is not eligible."
-us,scenario_117,federal_income_tax_before_refundable_credits,50,prompt_ambiguity,taxable_income_or_deductions,False,,"This case tests 2026 law after OBBBA. AGI is $250,733.36: wages, plus the $2,589 refund, minus the $530 loss. From that, three deductions apply: the $32,200 joint standard deduction, the full $25,000 qualified overtime deduction, and the $2,000 charitable deduction for non-itemizers. The overtime deduction is not reduced because the joint phase-out starts at $300,000 MAGI. Itemizing does not beat this $34,200 combination. Taxable income is $191,533.36. The permanent TCJA brackets (10% to $24,800, 12% to $100,800, then 22%) give $31,561.34 of tax. The only nonrefundable credit is the $6,600 CTC ($2,200 × 3 children under 17). The CDCC is $0 because Head has no earned income, and mortgage interest is $0 because only a loan balance is given. Most wrong answers left out the overtime deduction or reduced it, applied TCJA-sunset rules, made up mortgage interest, claimed a CDCC, or skipped the $2,000 charitable deduction. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,961.34; the alternative is 24,391.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
+us,scenario_117,federal_income_tax_before_refundable_credits,51,prompt_ambiguity,taxable_income_or_deductions,False,,"The reference uses AGI of $250,733.36: wages, plus the taxable $2,589 refund, minus the $530 loss. From that it subtracts $59,200: the $32,200 2026 joint standard deduction, the full $25,000 joint overtime-premium deduction (its phase-out starts at $300,000 MAGI for joint filers, not $150,000), and the $2,000 joint non-itemizer charitable deduction. That leaves taxable income of $191,533.36 and tax of $31,561.34. The only nonrefundable credit is the full $6,600 CTC (3 × $2,200). There is no CDCC because Head has no earned income. Most wrong answers had one of these errors: they left out or cut the overtime deduction, itemized against $32,200 instead of the larger $34,200 standard-deduction path, claimed a CDCC, used pre-OBBBA or TCJA-sunset parameters, or imputed mortgage interest from a balance when no interest amount was given. Developer adjudication (2026-09-29): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 24,961.34; the alternative is 24,391.80. The prompt lists state and local tax refund income without saying whether the refunded tax reduced federal tax in the year it was deducted; policyengine-us 2.15.17 counts the whole refund as income (policyengine-us#9422, fixing issue #9122), while under 26 U.S.C. 111(a) none of it is income if the household took no tax benefit from the deduction. Found in the 2026-09-29 engine upgrade (reference_audit/2026-09-28, cluster salt_refund_gross_income_9122). Reference depends on an unlisted input; output excluded from scoring (whether the prior-year deduction of the refunded state and local tax reduced federal tax (prior-year itemization, the income-versus-sales-tax election, SALT-cap headroom))."
us,scenario_117,federal_refundable_credits,5,llm_error,other,False,,"The refundable CTC (ACTC) is capped at the part of the child tax credit left over after the credit has offset income tax. This couple files jointly with about $250,733 of AGI. That is below the $400,000 CTC phase-out, so the full $2,200-per-child CTC ($6,600) is allowed. Their federal income tax is well over $25,000, which absorbs the whole $6,600 as a nonrefundable credit and leaves nothing to refund. EITC is zero at this income, and no other refundable credit applies, so the total is $0. The wrong answers skipped the tax-liability limit and treated the per-child refundable cap ($1,700 or $1,800), or a made-up fraction of it, as money the family receives automatically."
-us,scenario_117,free_school_meals_eligible,51,llm_error,state_local_rule,False,,"On income alone, the household is in the paid tier. Its $248,675 in wages is 6.43 times the 5-person poverty guideline, and it has no SNAP or TANF categorical eligibility. Arkansas Act 123 of 2025 (A.C.A. § 6-18-722(f)) separately gives every public school student one free breakfast a day starting in the 2025-26 school year, whatever the family income. PolicyEngine assumes the two K-12 children (ages 16 and 11) take it, which is worth $2.06 per child per day over 180 days, or $741.60 a year. Because free school meal support is positive, the answer is Yes. Every model ruled out free school meal support and missed this Arkansas universal-breakfast rule."
+us,scenario_117,free_school_meals_eligible,52,llm_error,state_local_rule,False,,"Under the federal income test this household is in the paid tier. Its $248,675 in wages is 6.43 times the $38,680 guideline for five people, and it has no SNAP or TANF categorical eligibility. Every model correctly stopped free lunch at that point. The step every model missed is Arkansas Act 123 of 2025 (A.C.A. § 6-18-722(f)). Starting in the 2025-26 school year, it gives every public school student who doesn't already get a free breakfast one free breakfast a day, regardless of income. For the two K-12 children (ages 16 and 11), that breakfast is worth $2.06 a day × 180 days × 2 = $741.60, which is positive free school meal support, so the answer is 1."
us,scenario_117,head_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is individual, not household-wide: living with a two-year-old does not make the 44-year-old head categorically eligible. The household's approximately $250,733 annual income also exceeds the WIC limit of 185% of the federal poverty guideline for a five-person household."
us,scenario_117,payroll_tax,27,llm_error,thresholds_rates,False,,"Only the spouse has wages ($248,675, which already include overtime). Employee Social Security tax is 6.2% up to the 2026 wage base of $184,500, which comes to $11,439. Medicare is 1.45% of all wages, $3,605.79. Combined wages are under the $250,000 MFJ Additional Medicare threshold, and Arkansas has no employee payroll tax, so the total is $15,044.79. Most models used an outdated or guessed Social Security wage base, such as $168,600 (2024) or $176,100 (2025). Others counted the FLSA overtime premium twice, skipped the SS cap, or submitted a number that does not match their own arithmetic."
us,scenario_117,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_117,spouse_wic_eligible,1,llm_error,categorical_eligibility,False,,"WIC eligibility is determined individually and requires both membership in an eligible category—pregnant, postpartum, breastfeeding, infant, or child under five—and satisfaction of the income or adjunctive-eligibility rules. The spouse is not placed in any eligible WIC category by the stated facts, and household income of approximately $250,733 also exceeds the applicable 185% federal poverty guideline."
-us,scenario_117,state_income_tax_before_refundable_credits,51,llm_error,thresholds_rates,False,,"The engine's Arkansas computation takes four steps. First, Arkansas AGI is $248,144.53: all wages, overtime included, less the $530 capital loss. The $2,589 state refund is excluded and there is no overtime exclusion. Second, the household itemizes $24,646.33, which is the $25,900 charitable gift less the engine's charitable-deduction floor of 0.5% of federal AGI; medical costs and employee expenses fall below their floors, and with no interest input there is no mortgage interest. Third, the engine's 2026 rate schedule gives $8,189.53 on $223,498.20 of taxable income. That is about a 3.66% effective rate, roughly $448 less than the old 2%/4%/3.9% high-income table (about $8,637). Fourth, $205 of nonrefundable credits come off: $145 in personal credits plus a $60 additional credit. The models closest to the answer deducted the full $25,900, used the 3.9% table and subtracted only $145. Models further off took the standard deduction, imputed mortgage interest from the balance, added the overtime premium on top of wages, used out-of-date 4.4%–4.9% top rates, or dropped the credits entirely."
+us,scenario_117,state_income_tax_before_refundable_credits,52,llm_error,thresholds_rates,False,,"Arkansas AGI is about $248,145: the spouse's wages, which already include the overtime, less the $530 capital loss. The $2,589 state tax refund is excluded. The only itemized deduction is charity, allowed at $24,646.33 ($25,900 minus 0.5% of the $250,734 federal AGI, or $1,253.67). Only a mortgage balance is given, so mortgage interest is $0, and medical costs fall below the AGI floor. The engine's 2026 Arkansas rate schedule produces $8,189.53 of tax on $223,498.20 of taxable income, and $205 of nonrefundable credits ($145 personal plus a $60 additional credit) brings it to $7,984.53. Most models made the same three errors. They deducted the full $25,900, used the 3.9%-top-rate high-income table (about $446 more tax than the engine's schedule), and subtracted only $145. Others skipped itemizing, imputed mortgage interest, added the overtime twice, or used old 4.4%–4.9% rates."
us,scenario_117,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_118,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"The head is 74, blind, and disabled. Their only income is $2,800 a year in Social Security, and they hold $70 in resources. That qualifies them for SSI: countable income after the $20 general exclusion is about $213/month, which is under the federal benefit rate, and the engine pays $9,368 a year. In New York, SSI receipt brings Medicaid eligibility automatically through the SSI-recipient category, and the head also meets NY's aged/disabled income and asset limits. Both models missed that the SSI pathway makes the head categorically eligible. They also failed to check the very low income and assets against the aged/blind/disabled limits."
us,scenario_118,payroll_tax,1,llm_error,payroll_tax_base,False,,"Employee payroll taxes apply to wages and other covered earned compensation, not Social Security retirement benefits. With no wages, self-employment income, or other covered earnings listed, the employee-side payroll-tax base is zero and payroll tax is $0."
-us,scenario_118,snap,51,prompt_ambiguity,taxable_income_or_deductions,False,,"The head is 74, blind, disabled and has $70 of assets, so they receive federal SSI of $780.67/month ($994 minus $213.33 of countable Social Security). Federal SSI, Social Security and the $87/month New York supplement give SNAP gross income of $1,101/month on policyengine-us 2.60.0. The $209 standard deduction leaves $892. Under the engine's default county, shelter costs are $136.17/month of property tax plus the $877 utility allowance; housing_cost does not include the listed mortgage interest. The uncapped shelter deduction is about $567.17, net income rounds to $325, and the contribution rounds up to $98, giving $200/month or $2,400/year under the publication conventions. The recorded $2,903.94 remains a frozen excluded value; the mortgage residence and county are unlisted inputs. Models that left out SSI landed at or near the maximum allotment (~$3,500). Models that counted SSI but used only mortgage interest and property tax as shelter costs landed near $1,384. Models that dropped the shelter deduction or ruled the household ineligible came in lower still, down to $0. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (whether the listed home mortgage interest is on the home the SNAP household occupies)."
+us,scenario_118,snap,52,prompt_ambiguity,taxable_income_or_deductions,False,,"Two inputs separate the reference from the wrong answers. First, the 74-year-old blind and disabled head receives federal SSI of $780.67/month ($994 SSI rate minus $213.33 of countable Social Security). SNAP counts that SSI as unearned income, so gross income is $1,014/month, not $233. Second, the uncapped elderly/disabled shelter costs are not just mortgage interest plus property tax ($598.75/month). They also include New York's standard utility allowance (SUA), which brings total shelter costs to about $1,013/month. Against $805 of adjusted income (after the $209 standard deduction), that gives a $610.62 excess shelter deduction, $194.38 of net income and a $298 − $58.20 = $239.80 monthly allotment. Nine months at $239.80 plus three months at $248.58 gives $2,903.94 a year. Models that dropped SSI landed at the maximum allotment (about $3,500). Models that counted SSI but left out the SUA landed near $1,384. Models that also dropped the mortgage and tax shelter costs landed near $678. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 2,903.94; the alternative is 3,576.00. The prompt lists home mortgage interest without saying the mortgaged home is the household's residence; if it is, SNAP counts the mortgage payment, interest included, as a shelter cost (7 CFR 273.9(d)(6)(ii)(A)), which is the reading under the alternative value. Reference depends on an unlisted input; output excluded from scoring (whether the listed home mortgage interest is on the home the SNAP household occupies)."
us,scenario_118,ssi,38,llm_error,state_local_rule,False,,"The `ssi` output is the federal SSI payment only. Take the 2026 federal benefit rate of $994/month and subtract countable unearned income of $213.33/month ($2,800/12 = $233.33 in Social Security, minus the $20/month general exclusion). That leaves $780.67/month, which is $9,368/year. The New York State Supplement is a separate variable and is not part of `ssi`. The wrong answers fall into four groups: adding a NY living-alone supplement of about $87/month ($1,044/year) on top, using a stale or wrongly projected benefit rate ($943, $967, $990, $991, $982.67), taking the $20 exclusion once a year instead of every month, or calling the person ineligible by comparing $2,800 of annual income against a monthly benefit rate."
us,scenario_118,state_refundable_credits,39,llm_error,state_local_rule,False,"PolicyEngine uses NY AGI as household gross income for the real property tax credit, so the $2,800 of nontaxable Social Security is left out. NY Tax Law §606(e) and Form IT-214 (line 10) count that Social Security in household gross income. Counting it puts the household in the $2,001–$3,000 band, where the age-65+ maximum is $341, not the $375 maximum for $1,000 or less.","The key step is recognizing New York's real property tax credit (Tax Law §606(e), IT-214) as a refundable credit for low-income homeowners. It needs no tax liability, earned income, children or rent. This 74-year-old homeowner has federal AGI of $0 and property taxes of $1,634, above 3.5% of that AGI, so the flat table in force from 2025 gives $375. Most models checked only for the EITC and child credits and returned $0. Other models used the repealed household-gross-income rule, applied an incorrect credit formula or amount, or added the nonrefundable household credit. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated llm_error (state_local_rule). From 2025 New York keys the real property tax credit to federal AGI and a flat table; with federal AGI of $0 and property tax above 3.5% of it, a filer 65 or older receives $375. The judge applied the repealed household-gross-income rule. Reference affirmed (N.Y. Tax Law 606(e) as amended by Part RR of Chapter 59 of the Laws of 2025)."
-us,scenario_119,child1_chip_eligible,19,llm_error,taxable_income_or_deductions,False,,"Child 1 is 14 and passes CHIP's age test, and the child is not Medicaid-eligible (category NONE). The engine counts $52,569 in household MAGI after subtracting the $3,087 traditional 401(k) contribution and $144 IRA deduction from $55,800. This income is within Virginia's FAMIS limit. The child's employer-sponsored insurance disqualifies the child from CHIP, so is_chip_eligible is False. The models that answered eligible missed this coverage restriction."
+us,scenario_119,child1_chip_eligible,20,llm_error,health_coverage,False,,"Child 1 is 14 and passes CHIP's age test. The household's MAGI is above Virginia's child Medicaid limit, so the child is Medicaid category NONE, and the income is close to the FAMIS limit. The fact that decides the case is that the question lists Child 1 as having employer-sponsored insurance. A CHIP 'targeted low-income child' must be uncovered (no group health plan or other creditable coverage), so PolicyEngine fails the child on CHIP's own categorical test and returns not eligible. Every model that answered only compared income with the roughly 200–205% FPL FAMIS limit and either ignored the ESI flag or explicitly treated it as no bar to eligibility."
us,scenario_119,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,child1_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,child1_medicaid_eligible,5,llm_error,health_coverage,False,,"The separating step is the boundary between Virginia's Medicaid and its separate CHIP program. A 14-year-old with no SSI, no disability, and no other categorical hook reaches Medicaid only through the MAGI pathway for children ages 6-18, capped at 143% FPL plus the 5-point disregard; this household sits at 1.92x FPL after the $3,231 of traditional 401(k)/IRA contributions, so medicaid_category is NONE. Coverage from 148% up to about 200% FPL in Virginia is FAMIS, which PolicyEngine reports as CHIP under the separate child1_chip_eligible output, not as Medicaid. Every substantive wrong answer imported the FAMIS/CHIP threshold into the Medicaid question, and one also sized the household at four people instead of three."
us,scenario_119,child1_medicare_eligible,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,child1_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_119,child2_chip_eligible,19,llm_error,taxable_income_or_deductions,False,,"Child 2 is 11, so the age test passes. The engine rejects CHIP eligibility because Child 2 has employer-sponsored insurance, which is disqualifying coverage for CHIP. The income test passes. The engine counts MAGI of about $52,569 (wages of $55,000 less the $3,087 traditional 401(k) deferral, plus $800 of interest, less the $144 IRA deduction). That is about 192% of the 2026 3-person poverty guideline of $27,320 and within Virginia's 205% FAMIS limit. Child 2 is also above the 148% FPL Medicaid limit, so neither program covers them. Almost every model tested only Child 2's age and income, which Child 2 passes, and no model treated Child 2's employer-sponsored insurance as coverage that disqualifies them from CHIP."
+us,scenario_119,child2_chip_eligible,20,llm_error,taxable_income_or_deductions,False,,"Child 2 is 11, so the age test passes and only the income test matters. PolicyEngine counts the 3-person household's income at about $55,800 (wages plus interest). Measured against the poverty guideline, that is above Virginia's FAMIS (CHIP) child limit, and the child is not Medicaid-eligible either, so the result is not eligible. Almost every model first subtracted the traditional 401(k) and IRA contributions (and one also subtracted the ESI premiums) to get a MAGI of about $52,569 or less. That lower figure works out to roughly 190–197% FPL, which put the household under the 200%/205% FAMIS line and led to a ""Yes."""
us,scenario_119,child2_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,child2_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_119,child2_medicaid_eligible,5,llm_error,thresholds_rates,False,,"The trap is the boundary between Virginia's children's Medicaid MAGI limit and its FAMIS/CHIP ceiling: Medicaid for children ages 6-18 in Virginia stops at 143% FPL, while FAMIS covers 143% up to 200% FPL (205% with the 5% disregard). Child2's household MAGI lands at 1.92 x FPL, which sits squarely in the CHIP band and above every Medicaid MAGI child category, and with zero SSI, no disability, and no foster or medically-needy pathway the engine assigns medicaid_category NONE. Every substantive wrong answer imported the 200%-205% FAMIS ceiling as the Medicaid threshold, treating FAMIS coverage as Medicaid coverage; two models emitted no value at all for this key."
@@ -937,7 +937,7 @@ us,scenario_119,ssi,1,parse_contract_failure,missing_output,False,,All wrong res
us,scenario_119,state_income_tax_before_refundable_credits,50,llm_error,state_local_rule,False,,"To get Virginia taxable income of $39,468.54 and tax of $2,011.94, you start from federal AGI of $52,568.54 and take three subtractions: the 2026 standard deduction of $8,750, three $930 exemptions ($2,790), and the $1,560 child and dependent care expense deduction. Virginia then allows a nonrefundable EITC equal to 20% of the federal EITC ($1,276.30 → $255.26), which brings the result to $1,756.68. Most models missed one or more of these steps. They left out the nonrefundable VA EITC, skipped the $1,560 care deduction or swapped it for a nonexistent VA care credit, used an old $8,000/$8,500 standard deduction, or cut AGI a second time by subtracting ESI premiums already excluded from wages."
us,scenario_119,state_refundable_credits,18,llm_error,state_local_rule,False,,"Virginia's earned income credit is an either/or election. A filer takes the nonrefundable credit (20% of federal EITC) or the refundable credit, never both, and PolicyEngine counts the refundable version only when it beats the nonrefundable credit the filer can actually use. This head of household earns about $55,000, so Virginia tax liability is roughly $2,000 or more. That fully absorbs a nonrefundable credit worth a few hundred dollars, so the household takes the nonrefundable credit and va_refundable_credits is $0. Every model that answered added a refundable Virginia EITC on top of the regular credit, treating it as automatic and stackable instead of as the alternative to the nonrefundable credit."
us,scenario_119,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_120,federal_income_tax_before_refundable_credits,51,reference_engine_defect,taxable_income_or_deductions,False,,"The reference applies 2026 law after OBBBA. The TCJA brackets are permanent, with the 24% bracket running from $105,700 to $201,775. There is no personal exemption, and miscellaneous itemized deductions stay disallowed. The frozen computation used AGI of $232,635, including a $29 IRA deduction and excluding the $1,750 state tax refund. It used itemized deductions of $35,717 (charity $19,001, SALT $13,908 and mortgage interest $2,808), taxable income of $196,918, regular tax of $39,858 and NIIT of $163.40. The policyengine-us 2.60.0 engine includes the refund and applies the IRA phase-out, yielding $40,423.18. The output remains excluded because NIIT inclusion and state income tax paid are unstated. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 40,021.82; the corrected value is 40,028.32. The frozen engine deducted traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
+us,scenario_120,federal_income_tax_before_refundable_credits,52,reference_engine_defect,taxable_income_or_deductions,False,,"The reference applies 2026 law under the OBBBA. The TCJA 10/12/22/24/32% brackets are permanent, with the 24% bracket running from $105,700 to $201,775 for a single filer. The SALT cap is $40,400, so the full $13,908 of Connecticut income tax plus property tax is deductible. Mortgage interest is limited to the $750k/$1M share ($2,808), charitable gifts face a 0.5%-of-AGI floor ($19,001), unreimbursed employee expenses stay non-deductible, and the $6,000 senior deduction phases out completely. AGI is $232,635, which leaves out the $1,750 state tax refund. Itemized deductions of $35,717 bring taxable income to $196,918, and the regular tax is $39,858.42. A 3.8% NIIT on $4,300 of interest plus gross rental income (not netted against the farm rent loss) adds $163.40. The wrong answers mostly fall into a few groups: they assumed the TCJA had expired, used the old $10k SALT cap, left out the state income tax, subtracted the employer health premiums from the given gross wages, skipped the charitable floor, taxed the state refund, or left out or shrank the NIIT. Developer adjudication (2026-09-22): the judge (claude-opus-5-5) returned llm_error; adjudicated reference_engine_defect (taxable_income_or_deductions). The frozen reference is 40,021.82; the corrected value is 40,028.32. The frozen engine deducted traditional IRA contributions without the active-participant phase-out. Reference is an engine defect; output excluded from scoring (26 U.S.C. 219(g); IRS Notice 2025-67 (2026 ranges))."
us,scenario_120,federal_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,free_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
@@ -945,12 +945,12 @@ us,scenario_120,head_medicaid_eligible,1,parse_contract_failure,missing_output,F
us,scenario_120,head_medicare_eligible,2,llm_error,age_disability,False,,"Medicare eligibility applies because the head is age 76, well above the age-65 threshold. Income, assets, disability status, and employer-sponsored insurance do not negate age-based Medicare eligibility."
us,scenario_120,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_120,payroll_tax,38,llm_error,state_local_rule,False,,"The reference has three parts: 6.2% Social Security on the full $165,597 of wages, which is below the 2026 wage base ($10,267.01), 1.45% Medicare ($2,401.16), and Connecticut's mandatory 0.5% employee Paid Leave contribution ($827.98). The total is $13,496.16, and no Additional Medicare Tax applies because wages are under $200,000. Most models stopped at the FICA total of $12,668.17 and wrongly said Connecticut has no employee payroll tax. A second group did include the CT paid leave contribution but took the $7,429 employer-sponsored insurance premium (and in some cases 401k deferrals) out of FICA wages. Neither the premium input nor 401k deferrals reduce the payroll tax base here."
+us,scenario_120,payroll_tax,39,llm_error,state_local_rule,False,,"The correct total is employee Social Security tax (6.2% × $165,597 = $10,267.01, under the 2026 wage base), plus employee Medicare tax (1.45% = $2,401.16), plus Connecticut's mandatory 0.5% Paid Leave employee contribution on the full wages ($827.98), for $13,496.16. There is no Additional Medicare Tax because wages are under $200,000. Most models stopped at the FICA-only $12,668.17 because they left out the CT paid-leave contribution. A second group cut the FICA base by subtracting the ESI premium and/or 401(k) deferrals from gross wages, but PolicyEngine taxes the full gross wages."
us,scenario_120,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,self_employment_tax,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,snap,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_120,state_income_tax_before_refundable_credits,51,llm_error,state_local_rule,False,,"Most models went wrong on Connecticut's Social Security subtraction. For a single filer with AGI over $75,000, the subtraction equals federally taxable benefits minus 25% of total benefits ($35,615 − $10,475 = $25,140). Subtracting that and the $1,750 state refund from federal AGI of $234,414 gives CT AGI and taxable income of $207,524, because the personal exemption, personal credit, pension exemption and property-tax credit are all fully phased out at this income. The 2024+ brackets (2%/4.5%/5.5%/6%/6.5%) give $11,239.06 of graduated tax, and adding the $250 2% bracket phase-out add-back plus $430 of recapture ($250 low-tax + $180 middle-tax) gives $11,919.06. The wrong answers mostly fell into one of four groups: they denied the SS subtraction entirely, subtracted all taxable SS, used the pre-2024 3%/5% rates, or left out some or all of the $680 in add-backs."
+us,scenario_120,state_income_tax_before_refundable_credits,52,llm_error,state_local_rule,False,,"The engine computes Connecticut AGI as federal AGI of $234,414 minus two subtractions. The first is a $25,140 Social Security subtraction: for a single filer above $75,000, federally taxable benefits of $35,615 are reduced to 25% of the $41,900 benefit, or $10,475. The second is the $1,750 state refund subtraction. That gives $207,524. The 2026 schedule (2%/4.5%/5.5%/6%/6.5%) produces $11,239.06 on that amount. Adding the $250 phase-out of the 2% bracket, the $250 low-tax recapture and the $180 middle-tax recapture (two $90 steps above $200,000) gives $11,919.06. The models diverged on four steps. Some subtracted none of the Social Security benefit, some subtracted all of it, and some used the wrong base for the 25% cap. Others skipped the refund subtraction, used the pre-2024 3%/5% low brackets, dropped some or all of the $680 in add-backs, or applied exemptions, deductions or credits that are fully phased out or don't exist in Connecticut."
us,scenario_120,state_refundable_credits,2,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_120,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_121,federal_income_tax_before_refundable_credits,4,prompt_ambiguity,taxable_income_or_deductions,False,,"Two facts drive the $0: PolicyEngine's `disability_benefits` input is a non-taxable income source excluded from AGI, so AGI is wages only ($25,665), and the household's medical costs ($2,200 non-Medicare-Part-B premiums + $30,000 other medical + $400 over-the-counter = $32,600) generate an itemized medical expense deduction of $32,600 − 7.5% × $25,665 ($1,925) = $30,675. That itemized total exceeds AGI, so taxable income floors at $0 and there is no tax to reduce with nonrefundable credits. Every wrong model either taxed the disability benefits, ignored the medical itemized deduction, or both, leaving a positive 10%/12%-bracket residue. Developer adjudication (2026-09-22): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 875.34. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)."
@@ -960,7 +960,7 @@ us,scenario_121,head_medicare_eligible,17,llm_error,age_disability,False,,"Polic
us,scenario_121,local_income_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_121,payroll_tax,7,llm_error,payroll_tax_base,False,,"The listed employer-sponsored insurance premium is not specified as a pre-tax payroll deduction, so it does not reduce FICA wages under the prompt’s rule that unlisted statuses are false. Applying 6.2% Social Security and 1.45% Medicare to the full $25,665 of wages yields $1,591.23 and $372.14, totaling $1,963.37."
us,scenario_121,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_121,snap,34,llm_error,age_disability,False,,"The question is whether this household counts as an elderly/disabled SNAP household. PolicyEngine follows the USDA definition, which requires receipt of a qualifying disability program such as SSI or SSDI. A generic `is_disabled` flag plus generic ""disability benefits"" income does not meet it. So the gross income test applies, and the head's $39,909 gross income ($3,326/month from wages, disability benefits and child support) fails it. The household's gross income also exceeds SC's broad-based categorical eligibility (BBCE) ceiling of 130% of the applicable poverty guideline, so SNAP is $0. Every substantive model treated the head as SNAP-disabled and skipped the gross test. Most also took the uncapped excess medical deduction, which is only available to elderly/disabled households, and so reached roughly the maximum allotment."
+us,scenario_121,snap,35,llm_error,categorical_eligibility,False,,"To be an elderly or disabled SNAP member under 7 CFR 271.2 and PolicyEngine's is_usda_disabled, a person must receive a qualifying disability program benefit, such as SSI or Social Security disability. A general 'is disabled' flag plus generic 'disability benefits' does not qualify. Without that status, this household gets no exemption from the gross income test and no uncapped excess medical deduction. Countable gross income of about $3,326 a month is far above South Carolina's 130% FPL limit of about $1,696, so SNAP is $0. Every model that answered treated the head as SNAP-disabled, skipped the gross test, and used the roughly $30,000 in medical costs to cut net income to near zero."
us,scenario_121,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_121,state_income_tax_before_refundable_credits,8,prompt_ambiguity,taxable_income_or_deductions,False,,"South Carolina's tax base starts from federal taxable income, and this filer's deductible medical outlays total $32,600 ($30,000 other medical expenses + $2,200 health insurance premiums excluding Medicare Part B + $400 over-the-counter health expenses). Netted against the 7.5%-of-AGI floor (at most $2,903 on the largest possible AGI of $38,709), that leaves roughly $30,000 of itemized medical deduction — nearly double the $16,100 single standard deduction — which wipes out federal taxable income, and SC's subtraction for the disabled filer's disability income removes anything left, so SC taxable income never reaches the top of the zero-rate bracket and the tax is $0. Every wrong model defaulted to the standard deduction and treated the $30,000 medical line as an irrelevant household fact; several compounded that with invented SC rate schedules and a nonexistent personal exemption. The gap between $0 and every submitted number is entirely the unclaimed itemized medical expense deduction — no bracket or rate correction closes it. Developer adjudication (2026-09-22): the judge (claude-opus-5) returned llm_error; adjudicated prompt_ambiguity (taxable_income_or_deductions). The frozen reference is 0.00; the alternative is 212.23. The prompt lists disability benefits from employment without saying who paid for the coverage. Benefits from employer-paid coverage are included in gross income (26 U.S.C. 105(a)); benefits from coverage the employee paid for with after-tax money are excluded (26 U.S.C. 104(a)(3)). The reference excludes them; the alternative value includes them. Reference depends on an unlisted input; output excluded from scoring (who paid for the coverage behind listed employment disability benefits, which decides whether they are taxable)."
us,scenario_122,federal_income_tax_before_refundable_credits,44,llm_error,taxable_income_or_deductions,False,,"AGI is $70,646, which includes 85% of Social Security ($27,200). Because the OBBBA keeps the TCJA structure in place for 2026, total deductions are $27,200: the $16,100 standard deduction, plus 2 × $2,050 for being 65+ and blind, plus the $6,000 senior deduction (MAGI is under $75,000), plus the new $1,000 charitable deduction for people who don't itemize. Taxable income is $43,446.18. The $1,824 of qualified dividends is taxed at 0%, and the remaining $41,622.18 is taxed at 10% up to $12,400 and 12% after that, for a total of $4,746.66. The wrong answers fall into four groups: models that left out the $1,000 charitable deduction (about $120 too high), models that left out the $6,000 senior deduction, models that used the old TCJA-expiration rules (personal exemption, 15% bracket, miscellaneous itemized deductions), and models that made unsupported jumps to $0 or to unrelated figures."
@@ -968,28 +968,28 @@ us,scenario_122,federal_refundable_credits,1,parse_contract_failure,missing_outp
us,scenario_122,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Age, blindness, and disability do not by themselves establish Medicaid eligibility under Minnesota’s aged, blind, or disabled pathways. The head qualified through no Medicaid category, received no SSI, and had MAGI equal to 4.85 times FPL, so the eligibility result is No."
us,scenario_122,payroll_tax,2,llm_error,payroll_tax_base,False,,"Employee payroll tax applies to wage and salary earnings, not Social Security benefits, IRA distributions, pensions, dividends, interest, or capital gains. Because the prompt lists no wages and requires unlisted numeric inputs to be zero, the employee payroll-tax base is zero and payroll tax is $0."
us,scenario_122,self_employment_tax,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_122,state_income_tax_before_refundable_credits,51,llm_error,state_local_rule,False,,"Minnesota tax starts from federal AGI of $70,646. It then subtracts the full $27,200 of federally taxable Social Security, because AGI is well below the phase-out threshold. Next comes the 2026 standard deduction of $19,300, which includes two aged/blind add-ons and exceeds the available itemized deductions. Finally, the non-itemizer charitable subtraction is $4,977, which is 50% of contributions over $500 after the 2026 0.5%-of-AGI floor. That leaves $19,169 taxed entirely at 5.35%, or $1,025.54. Minnesota's pension subtraction applies only to public pension income, so it does not reach this filer's private pension or IRA income, no personal exemption for the filer, and no elderly subtraction at this income. Most models went wrong in one of three ways: they skipped or only partly applied the Social Security subtraction (answers near $2,400–$3,200), left out the non-itemizer charitable subtraction (answers near $1,250–$1,400), or invented exclusions or credits that brought the tax to $0."
-us,scenario_122,state_refundable_credits,9,llm_error,credit_phaseout,False,,"Eight models awarded Minnesota's renter's credit, and kimi-k2.6 returned no value. The engine uses federal AGI and a $5,300 age-based subtraction for this filer, giving renter's-credit household income of $65,346, below the 2026 schedule ceiling of $79,330. The credit also requires a qualifying Certificate of Rent Paid (CRP), an input the prompt does not list and the engine defaults to false, so mn_refundable_credits is $0; with a qualifying CRP the engine pays $1,921. The household also has no earned income for the Working Family Credit and no children for the MN Child Tax Credit. The eight models computed a credit from rent and household income, and none addressed the CRP requirement. The prompt tells models to treat an unlisted boolean as false, which gives the engine's reading."
+us,scenario_122,state_income_tax_before_refundable_credits,52,llm_error,state_local_rule,False,,"Minnesota starts from federal AGI of $70,646.18. Three Minnesota rules then bring taxable income down to $19,169, which falls entirely in the 5.35% bracket and gives $1,025.54. First, the full $27,200 of taxable Social Security is subtracted, because AGI is below the phaseout. Second, the filer takes the $19,300 single standard deduction, which includes the aged and blind additions. Third, the non-itemizer charitable subtraction of $4,977.18 applies. Models that skipped or partly phased out the Social Security subtraction put income into the 6.8% bracket (about $2,400–$3,200). Models that left out the charitable subtraction landed around $1,250–$1,400. Models that invented senior, pension or elderly/disabled relief reached $0 or very low amounts. The models closest to the reference overstated the standard deduction or used $5,154 for the charitable subtraction instead of $4,977.18."
+us,scenario_122,state_refundable_credits,9,prompt_ambiguity,state_local_rule,False,,"Eight models awarded Minnesota's renter's credit, and kimi-k2.6 returned no value. The engine uses federal AGI and a $5,300 age-based subtraction for this filer, giving renter's-credit household income of $65,346, below the 2026 schedule ceiling of $79,330. The credit also requires a qualifying Certificate of Rent Paid (CRP), an input the prompt does not list and the engine defaults to false, so mn_refundable_credits is $0; with a qualifying CRP the engine pays $1,921. The household also has no earned income for the Working Family Credit and no children for the MN Child Tax Credit. The eight models computed a credit from rent and household income, and none addressed the CRP requirement. The prompt tells models to treat an unlisted boolean as false, which gives the engine's reading. Developer adjudication (2026-11-01): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (state_local_rule). The frozen reference is 0.00; the alternative is 1,921.00. The prompt places the household in Minnesota with pre-subsidy rent of $28,235 and states no Certificate of Rent Paid (CRP); policyengine-us 2.60.0 grants Minnesota's renter's credit only with a qualifying CRP (mn_renters_credit_qualifying_crp, a boolean input that defaults to false), while Minn. Stat. 290.0693 subd. 4 requires the owner or managing agent of a rented homestead to furnish one to a renter on December 31; read as a renter who holds the CRP, the household's renter's credit household income of $65,346.18 falls under 2026's $79,330 first zero threshold, and the credit, and with it the output, is $1,921. The 2026-10-29 audit traced it (reference_audit/2026-10-29-mn-renters-crp), and Max's decision d2180 excludes it. Reference depends on an unlisted input; output excluded from scoring (a qualifying Certificate of Rent Paid (mn_renters_credit_qualifying_crp))."
us,scenario_123,child1_chip_eligible,4,llm_error,thresholds_rates,False,,"Child 1 is 16, so the CHIP age rule (under 19) is met. What decides the case is the income test. The child has to file a return on $45,000 of wages, so that income counts in household MAGI. With the parents' income, household MAGI is about $145,002, which is over 500% FPL for a family of three. That is far above Pennsylvania's CHIP income ceiling (314% FPL), and PolicyEngine's is_chip_eligible applies that ceiling and does not treat the unsubsidized full-cost CHIP buy-in as eligibility. Every model that answered 'Yes' skipped the income ceiling: either it assumed PA's full-cost CHIP makes any income eligible, or it looked only at the child's age."
us,scenario_123,child1_early_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,child1_head_start_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,child1_medicaid_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,child1_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,child1_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_123,federal_income_tax_before_refundable_credits,50,prompt_ambiguity,household_unit_or_filing_status,False,,"The frozen reference covers only the parents' joint tax unit. The requested output is excluded from scoring because the prompt leaves return scope unspecified. AGI is $100,002 because the dependent child's $45,000 of wages is not on the parents' return and no separate tax on the child is added. Under 2026 OBBBA law, deductions are the $32,200 MFJ standard deduction plus the new $2,000 non-itemizer cash charitable deduction, which leaves $65,802 of taxable income. Tax is $7,400.24, and subtracting the $2,200 nonrefundable CTC gives $5,200.24. The overtime deduction reads the explicit FLSA overtime-premium input, which is $0 when unlisted. Most models differed from the frozen reference in one of four ways: they put the child's wages on the joint return, added the child's own separate tax, used pre-TCJA sunset rules, or left out the $2,000 charitable deduction. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 5,200.24; the alternative is 8,420.24. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
+us,scenario_123,federal_income_tax_before_refundable_credits,51,prompt_ambiguity,household_unit_or_filing_status,False,,"The reference is the parents' joint tax unit only. AGI is $100,002 because the dependent child's $45,000 of wages does not enter the joint AGI, and no separate tax for the child is added. Taxable income is $65,802: AGI minus the 2026 OBBBA standard deduction of $32,200 and the new $2,000 non-itemizer charitable deduction for joint filers. Tax on that is $7,400.24, and the $2,200 nonrefundable CTC brings it to $5,200.24. The models mostly went wrong in one of these ways: they put the child's wages into the joint AGI, added a separate child return, left out the $2,000 non-itemizer charitable deduction, applied pre-TCJA sunset law or 2025 parameters, or made up mortgage interest or an overtime deduction that the facts don't support. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 5,200.24; the alternative is 8,420.24. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
us,scenario_123,federal_refundable_credits,7,llm_error,credit_phaseout,False,,"The Child Tax Credit is a nonrefundable credit first; the additional child tax credit is only the residual the taxpayer cannot use against liability, further capped at $1,700 per child in 2026 and at 15% of earnings above $2,500. This household's $145,002 of income less the 2026 MFJ standard deduction leaves roughly $112,800 of taxable income and about $14,000 of federal income tax before refundable credits, which absorbs the entire CTC for the 16-year-old, and the one-child EITC is long gone at joint income above $57,000 — so total federal refundable credits are $0. Every wrong model stopped at the earned-income phase-in and the $400,000 MAGI phase-out and paid out a per-child refundable cap ($1,500, $1,600, $1,700, $2,000, or $2,400) as if the ACTC were a standalone entitlement. One model additionally fabricated a $205 EITC at an income level far past its phase-out."
-us,scenario_123,free_school_meals_eligible,51,llm_error,state_local_rule,False,,"Income of $145,002 is 5.31 times the $27,320 FPG for three, so both meals land in the federal PAID tier and the 130%/185% income tests never qualify this household. The trap is Pennsylvania's universal free breakfast (Act 33 of 2023, 24 P.S. § 13-1337.1(c.2)). It makes breakfast free for the 16-year-old K-12 student at any income, worth $2.06/day ($2.90 subsidy minus $0.84 paid rate) × 180 days = $370.80. Positive free-meal support yields 1. Every model that answered stopped at the federal NSLP income test and never applied the state's universal breakfast."
+us,scenario_123,free_school_meals_eligible,52,llm_error,state_local_rule,False,,"The household's $145,002 income is 5.31 times the poverty guideline, so on income alone both breakfast and lunch fall in the paid tier, and every model stopped there. Pennsylvania's Act 33 of 2023 (24 P.S. § 13-1337.1(c.2)) gives a free breakfast to students at participating schools regardless of household income. PolicyEngine places the 16-year-old K-12 student's breakfast in the free tier and values it at $370.80 a year ($2.06 a day over 180 school days). Because the output asks whether any free school meal support is positive, the answer is 1. All of the models that answered applied only the federal 130% FPL lunch test and missed the state's universal breakfast."
us,scenario_123,head_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,head_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Pennsylvania Medicaid eligibility requires both an applicable categorical pathway and satisfaction of that pathway's financial test. The 39-year-old head has no qualifying Medicaid category, and household MAGI is 5.31 times FPL, so the head fails both the categorical and income requirements."
us,scenario_123,head_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,head_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,local_income_tax,3,llm_error,state_local_rule,False,,"The household lists only state PA. No county or city is given, so it is not placed in Philadelphia. The local_income_tax output covers only NYC income tax, Philadelphia wage tax, Kansas City earnings tax and St. Louis earnings tax. A PA household outside Philadelphia gets none of these, so the value is $0. All three models assumed a Philadelphia residence (or a general PA municipal EIT) that the prompt never states, even though it says to treat unlisted facts as false and not to infer them, and taxed the $145,000 in household wages."
-us,scenario_123,payroll_tax,38,llm_error,state_local_rule,False,,"The two earners have $145,000 in combined wages. Each earner is below the Social Security wage base and below the Additional Medicare Tax threshold, so federal FICA is 7.65% × $145,000 = $11,092.50. The step that separates the reference from most answers is Pennsylvania's mandatory employee Unemployment Compensation contribution: 0.07% on all gross wages with no cap, or $70 + $31.50 = $101.50. Adding it gives $11,194. Most models said Pennsylvania has no employee payroll tax. A few models included the contribution but with the wrong rate or base, or they computed $11,194 and then submitted a different number."
+us,scenario_123,payroll_tax,39,llm_error,state_local_rule,False,,"The household has $145,000 of wages, all below the Social Security wage base and the Additional Medicare Tax thresholds. Federal FICA is therefore 7.65% × $145,000 = $11,092.50. The question asks for mandatory employee state payroll taxes, and Pennsylvania withholds an employee unemployment compensation contribution of 0.07% on all gross wages with no wage cap. That adds 0.0007 × $145,000 = $101.50, for a total of $11,194. Most models stopped at $11,092.50 because they believed Pennsylvania has no employee payroll tax. The rest capped the UC base, used an old rate, invented an Additional Medicare Tax, made arithmetic errors, or submitted a number that did not match their own correct working."
us,scenario_123,reduced_price_school_meals_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_chip_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_medicaid_eligible,2,llm_error,categorical_eligibility,False,,"Pennsylvania Medicaid eligibility requires qualification through a specific categorical pathway and its applicable income test. The 39-year-old spouse has no qualifying category and MAGI of 5.31 times FPL, so the spouse fails every available pathway and is not eligible."
us,scenario_123,spouse_medicare_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,spouse_wic_eligible,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
-us,scenario_123,state_income_tax_before_refundable_credits,50,prompt_ambiguity,household_unit_or_filing_status,False,,"The prompt puts everyone in one tax unit: Head and Spouse file jointly, and the 16-year-old is their dependent. The dependent's $45,000 of wages is not part of the unit's income (irs_gross_income = $100,002), so PA's flat 3.07% applies only to $100,000 of wages plus $2 of interest, which gives $3,070.06. PA has no standard deduction, itemized deductions or forgiveness that changes this. Almost every model either added the child's $45,000 to the base or added a separate $1,381.50 child return, which produces roughly $4,451.56. A few others applied deductions that PA does not allow. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 3,070.06; the alternative is 4,451.56. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
+us,scenario_123,state_income_tax_before_refundable_credits,51,prompt_ambiguity,household_unit_or_filing_status,False,,"The question defines one tax unit: Head, Spouse and their dependent Child 1. Pennsylvania's flat 3.07% applies only to the joint return's compensation and interest: $100,000 wages + $2 interest = $100,002, and $100,002 × 0.0307 = $3,070.06. Pennsylvania has no standard deduction, exemptions or itemized deductions, and Tax Forgiveness does not apply at this income. The dependent's $45,000 of wages goes on the child's own return, not the parents' return, so it is not part of the tax unit's PA taxable income. Nearly every model added the child's wages to the base ($145,002 → about $4,451.56) or added a separate child return to the total. A few others invented deductions, used the wrong rate or base, or made arithmetic slips. Developer adjudication (2026-10-06): the judge (claude-opus-5-5) returned llm_error; adjudicated prompt_ambiguity (household_unit_or_filing_status). The frozen reference is 3,070.06; the alternative is 4,451.56. A dependent with $45,000 of wages must file their own federal and state return, and the engine computes only the head and spouse's return. The output definitions name no return, tax unit or household, the prompt shows one tax unit and asks for quantities 'for this household', and careful readers split both ways (reference_audit/2026-10-05-reference-adversary/verification/independent/pa_123.md, verification/definition_conformance.md). Excluded under decision d1022 until decision d1029 settles the scope for version 2. Reference depends on an unlisted input; output excluded from scoring (whether the income tax outputs cover only the head and spouse's return or every return the household's members must file; the definitions name no return, and a dependent here must file their own)."
us,scenario_123,state_refundable_credits,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
us,scenario_123,tanf,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions.
diff --git a/app/public/paper/policybench.pdf b/app/public/paper/policybench.pdf
index 415928a6..f3906725 100644
Binary files a/app/public/paper/policybench.pdf and b/app/public/paper/policybench.pdf differ
diff --git a/app/public/paper/web/figures/positive_zero_scatter.png b/app/public/paper/web/figures/positive_zero_scatter.png
index 476301ae..9d2158c8 100644
Binary files a/app/public/paper/web/figures/positive_zero_scatter.png and b/app/public/paper/web/figures/positive_zero_scatter.png differ
diff --git a/app/public/paper/web/index.html b/app/public/paper/web/index.html
index 09c69634..a9682651 100644
--- a/app/public/paper/web/index.html
+++ b/app/public/paper/web/index.html
@@ -151,7 +151,7 @@
Table of contents
Abstract
-PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine populace microdataset and scoring answers against reference outputs from policyengine-us 2.60.0. It evaluates 51 frontier models on 100 households across 18 output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: 85% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of 69.6% weighted exact by 25.5 points (Section 3.1 details the construction). The manuscript snapshot is a 100-household public preview with no protected held-out set: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. In the frozen snapshot used for this manuscript (2026-10-27), the top-scoring US model on exact match is GPT-6 Sol at 95.2%, while GPT-5.4 nano is lowest at 65.3%. Within model families, newer is not always stronger: Claude Opus 4.8 scores 75.8% exact, below the 80.5% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover 9,050 rows whose legacy threshold score is below 1. That universe contains 9,046 of the snapshot’s 9,046 exact-match misses and 4 exact hits; 2,407 additional rows with bounded score below 100 were not selected or annotated. Every reference-suspect flag was adjudicated against primary sources and engine traces. PolicyBench rebuilt the references on 2026-10-23 with policyengine-us 2.60.0, the newest release when it began sweeping them that day. Each scored reference follows law published before PolicyBench froze the references on 2026-07-03. Where the engine projects a 2026 amount or takes one from a later publication, the reference holds the amount published before the freeze or, where none was published, the last amount published. PolicyBench excludes from scoring, for every model, the 47 outputs whose references depend on an input the prompt does not state and the two whose references rest on an amount a state published after the freeze. No excluded output rests on an engine defect the audits found that policyengine-us 2.60.0 still has. The 0.7% of cells with a missing or unparsable answer stay in the denominator as misses. The live benchmark is available at https://policybench.org.
+PolicyBench evaluates whether frontier language models can estimate household tax and benefit outputs from household facts without tools. This release covers the United States, drawing sampled households from the certified PolicyEngine Microcosm microdataset and scoring answers against reference outputs from policyengine-us 2.60.0. It evaluates 52 frontier models on 100 households across 18 output groups, and reports three complementary scores: an exact-match rate as the headline deployability bar (a prediction counts only if it matches the PolicyEngine reference to the dollar for amounts or to the eligibility flag for booleans), a within-1% hit rate as a near-miss-tolerant companion, and a continuous bounded score that awards partial credit for close answers. The headline is exact match on a household-impact-weighted leaderboard: 85% of US reference outputs are exact zeros, and the weighting concentrates scores in the high-dollar output groups a hedge-to-zero strategy cannot answer, so the top model leads an always-zero baseline of 69.6% weighted exact by 25.5 points (Section 3.1 details the construction). The manuscript snapshot is a 100-household public preview with no protected held-out set: the current public scenario explorer exposes prompts and reference outputs, so open-set leakage is a central limitation of any public ranking. In the frozen snapshot used for this manuscript (2026-11-11), the top-scoring US model on exact match is GPT-6 Sol at 95.2%, while GPT-5.4 nano is lowest at 65.3%. Within model families, newer is not always stronger: Claude Opus 4.8 scores 75.8% exact, below the 80.5% of the earlier Claude Opus 4.7. Multi-step tax quantities and positive-dollar benefit cases are materially harder than zero cases. The frozen audit annotations cover 9,132 rows whose legacy threshold score is below 1. That universe contains 9,128 of the snapshot’s 9,128 exact-match misses and 4 exact hits; 2,497 additional rows with bounded score below 100 were not selected or annotated. Every reference-suspect flag was adjudicated against primary sources and engine traces. PolicyBench rebuilt the references on 2026-10-23 with policyengine-us 2.60.0, the newest release when it began sweeping them that day. Each scored reference follows law published before PolicyBench froze the references on 2026-07-03. Where the engine projects a 2026 amount or takes one from a later publication, the reference holds the amount published before the freeze or, where none was published, the last amount published. PolicyBench excludes from scoring, for every model, the 48 outputs whose references depend on an input the prompt does not state and the two whose references rest on an amount a state published after the freeze. No excluded output rests on an engine defect the audits found that policyengine-us 2.60.0 still has. The 0.7% of cells with a missing or unparsable answer stay in the denominator as misses. The live benchmark is available at https://policybench.org.
Introduction
Household tax-and-benefit estimation sits between arithmetic and policy discussion. Each case has numeric labels, but generating those labels requires filing status, household composition, income concepts, program thresholds, and jurisdiction-specific rules. PolicyBench measures whether models can map household records to those outputs.
Most current language-model evaluations do not test this mapping directly. General math benchmarks emphasize symbolic manipulation and exact final answers (Cobbe et al. 2021; Hendrycks et al. 2021), while generic question-answering benchmarks emphasize recall or instruction following. PolicyBench instead asks whether models can transform household facts into policy outputs without access to tools or simulators.
@@ -163,11 +163,11 @@
The nearest public-benefits benchmark is the concurrent Public Benefits Bench, built by Vals AI with the Center for Civic Futures and Code for America (Vals AI 2026). It evaluates models on 459 expert-validated SNAP guidance scenarios (230 in the published test set) spanning all 50 US states plus Guam and the Virgin Islands, and grades free-text answers against expert-written rubric criteria using a judge model that agreed with SNAP policy experts on 80.6% of grading decisions. Where PolicyBench isolates parametric knowledge by withholding tools, Public Benefits Bench varies tool access directly: across models, multi-turn conversation raises pass rates by roughly 7.6 percentage points and web search by about 6.9. The two benchmarks probe complementary failure surfaces — conversational guidance quality for a single program with tools available, versus no-tool numeric estimation across tax and transfer outputs scored deterministically against microsimulation references — and reach consistent conclusions. The best model passes 71.7% of rubric criteria under Public Benefits Bench’s strongest condition, and its authors conclude that “no general-purpose AI model performs well enough to be trusted with SNAP benefits guidance,” matching the deployability gaps PolicyBench reports for unaided calculation.
Finally, PolicyBench depends on two infrastructure literatures that are not themselves large language model (LLM) benchmarks. One is structured-output reliability, since the benchmark relies on multi-output JSON responses and parse coverage (Shorten et al. 2024). The other is tax-benefit microsimulation, where systems such as EUROMOD provide the methodological precedent for evaluating policy rules over household microdata (Sutherland and Figari 2013). There is also operational work on applying and evaluating AI systems in public-benefits settings, including caseworker-assist and Supplemental Nutrition Assistance Program (SNAP)-focused evaluations (Nava Labs 2026, 2025; ZenML LLMOps Database 2025). PolicyBench combines these strands into a public cross-model benchmark over household-level tax and benefit outputs.
Benchmark design
-The canonical PolicyBench request asks each model to predict every benchmark output for a household in a whole-scenario response under its structured answer contract. Of the 51 models, 10 received the documented chunked accommodation and instead answered the same prompt template over one- or three-output subsets, reducing each response’s completion demand while preserving the household-level task.
+The canonical PolicyBench request asks each model to predict every benchmark output for a household in a whole-scenario response under its structured answer contract. Of the 52 models, 10 received the documented chunked accommodation and instead answered the same prompt template over one- or three-output subsets, reducing each response’s completion demand while preserving the household-level task.
Headline metric: exact match
The headline metric is the exact-match rate: the share of requested outputs the model gets right to the dollar (or, for eligibility flags, to the boolean). This is the deployability bar. A tax filer, a benefit estimator, or a caseworker cannot ship “close” — a prediction off by $50 is no more usable than one off by $500, because neither matches the bottom line a downstream system or a household needs. For currency amounts, “exact” means within 1 currency unit of the reference value after numeric parsing. For binary outputs, “exact” means the parsed value is exactly the same 0/1 flag as the reference.
Exact match can be a deceptive headline in a zero-inflated panel — but only when the rate is computed unweighted. In the frozen manuscript snapshot, 85% of reference outputs are exact zeros: most sampled households are not eligible for any given program, owe no capital gains tax, do not phase into a refundable credit, and so on. A model that hedges to zero on every requested output earns full exact credit on every zero case, so a naive unweighted exact rate is dominated by those zeros — a hedge-to-zero model scores roughly 85% unweighted, leaving little room above the floor to separate models that actually understand the rules.
-The public leaderboard corrects for this by household-impact weighting. Each output group’s weight is its population-average share of the household’s dollar stake, computed as |ref| / max(|household_net_income|, Σ |ref|) per source household and averaged with calibrated household weights over the full populace population (Section 5.5 gives the full construction). Zero-reference rows are not removed — every row carries its output group’s weight — but the weighting concentrates mass in high-dollar output groups, which are rarely zero, so the share of weight on zero-reference rows falls from their 85% row share to 69.6%. An always-zero baseline therefore scores 69.6% weighted exact, against the 85% it earns unweighted, and the top model clears that baseline by 25.5 points — a margin comparable to within-1%. Across the 51-model roster, the weighted exact rate spreads about as widely as within-1% and ranks models in nearly the same order, so exact match discriminates between rule comprehension and zero-hedging while preserving the to-the-dollar deployability meaning.
+The public leaderboard corrects for this by household-impact weighting. Each output group’s weight is its population-average share of the household’s dollar stake, computed as |ref| / max(|household_net_income|, Σ |ref|) per source household and averaged with calibrated household weights over the full Microcosm population (Section 5.5 gives the full construction). Zero-reference rows are not removed — every row carries its output group’s weight — but the weighting concentrates mass in high-dollar output groups, which are rarely zero, so the share of weight on zero-reference rows falls from their 85% row share to 69.6%. An always-zero baseline therefore scores 69.6% weighted exact, against the 85% it earns unweighted, and the top model clears that baseline by 25.5 points — a margin comparable to within-1%. Across the 52-model roster, the weighted exact rate spreads about as widely as within-1% and ranks models in nearly the same order, so exact match discriminates between rule comprehension and zero-hedging while preserving the to-the-dollar deployability meaning.
The exact rate is aggregated to a household score and then to a country score, with the same population-derived output weights that the within-1% companion and the bounded score use. Equal household weight preserves comparability across scenarios and slices. The leaderboard reports US results on this fixed test set.
Near-miss companion: within 1%
The within-1% hit rate is the near-miss-tolerant companion to the headline: the share of requested outputs the model gets within one percent of the PolicyEngine reference value. For currency amounts, “within 1%” means |pred − ref| ≤ 0.01 × |ref| when the reference is nonzero and |pred| ≤ 1 (the same one-currency-unit absolute tolerance used by exact match) when the reference is zero. Binary outputs are requested as integer 0/1 eligibility flags and scored identically to the exact-match metric.
@@ -179,8 +179,8 @@ Secondary me
Aggregation proceeds in three steps. First, each household-output prediction receives a 0-100 score. Second, each household receives one model score: requested output rows are weighted by output-group weights constructed from the full weighting population, then renormalized within the household over outputs that are actually requested for that household. Person-level coverage flags are scored at the person row; their output-group weight is split across the relevant people in the household and is based on PolicyEngine value proxies where available. Third, the country score averages those household scores with equal household weight. The leaderboard reports US results on this fixed test set. Equal-output-group scores and other alternative views are reported as sensitivity checks.
The benchmark requires each model response to include numeric answers and one explanation per requested output. Both the exact-match rate and the bounded score use only the numeric answers. Explanations are retained for scenario exploration and qualitative error analysis; they should not be interpreted as faithful traces of model reasoning.
Because explanations are required, the canonical task measures policy estimation under a public-facing structured-response contract, of which isolated arithmetic accuracy is one part. Prompt fairness is part of the benchmark contract. The current release uses one prompt template per country, with no model-specific tuning. Models receive the same household facts and requested outputs. They receive no web or external tool access. The prompt sets unlisted numeric inputs to 0, unlisted boolean or status facts to false, and household characteristics as constant over the tax-benefit year. Provider-specific differences are limited to structured-output transport, request shape, and the reasoning setup in Table 2. The harness sends no reasoning-control or sampling parameters. It leaves reasoning effort, temperature, and related decoding controls at provider defaults. The frozen GPT-5.5 wave uses provider-default reasoning effort.
-The forced tool_choice suppresses Claude’s extended thinking in this snapshot (sensitivity/claude-thinking-2026-08.md). A labeled sensitivity with tool_choice: "auto" is published beside the board. Claude Fable 5.1, released September 1, 2026, closes the interaction from the API side: it rejects forced tool use with an error, so its row answers through the JSON transport and reasons at the provider default; a sensitivity run with the tool declared under tool_choice: "auto" scores 0.8 points above that row. Claude Opus 5.5, released September 21, 2026, and Claude Sonnet 5.5, released September 28, 2026, reject forced tool use the same way, so their rows also answer through the JSON transport and reason at the provider default; neither has a sensitivity run. The next board version moves every model to tool_choice: "auto" so each provider’s default reasoning posture can engage.
-Two dimensions on which requests were not identical across models are request shape and answer transport (Table 2). Of the 51 models, 10 could not reliably complete the whole-scenario request because their serving stacks rejected the structured-output call, exhausted completion budgets mid-response, or timed out. They instead answered the same prompt template over subsets of the requested outputs, one to three outputs per request. Subsetting gives each response fewer requested outputs and a fresh completion budget over the same household facts, so the accommodation, if it moves scores at all, should favor the chunked models; their scores are not strictly comparable to whole-scenario scores. This accommodation is closed going forward: a model added after this snapshot either answers the canonical whole-scenario request or is listed as not scorable, without accommodation.
+The forced tool_choice suppresses Claude’s extended thinking in this snapshot (sensitivity/claude-thinking-2026-08.md). A labeled sensitivity with tool_choice: "auto" is published beside the board. Claude Fable 5.1, released September 1, 2026, closes the interaction from the API side: it rejects forced tool use with an error, so its row answers through the JSON transport and reasons at the provider default; a sensitivity run with the tool declared under tool_choice: "auto" scores 0.8 points above that row. Claude Opus 5.5, released September 21, 2026, Claude Sonnet 5.5, released September 28, 2026, and Claude Fable 5.2, released November 9, 2026, reject forced tool use the same way, so their rows also answer through the JSON transport and reason at the provider default; none has a tool_choice: "auto" sensitivity run. A separate sensitivity runs Claude Fable 5.2 and Claude Opus 5.5 once each at the other’s default effort (sensitivity/claude-effort-2026-11.md). The next board version moves every model to tool_choice: "auto" so each provider’s default reasoning posture can engage.
+Two dimensions on which requests were not identical across models are request shape and answer transport (Table 2). Of the 52 models, 10 could not reliably complete the whole-scenario request because their serving stacks rejected the structured-output call, exhausted completion budgets mid-response, or timed out. They instead answered the same prompt template over subsets of the requested outputs, one to three outputs per request. Subsetting gives each response fewer requested outputs and a fresh completion budget over the same household facts, so the accommodation, if it moves scores at all, should favor the chunked models; their scores are not strictly comparable to whole-scenario scores. This accommodation is closed going forward: a model added after this snapshot either answers the canonical whole-scenario request or is listed as not scorable, without accommodation.
One row, GLM-5.3-Flash (preview), answered as a cloaked OpenRouter listing called “Ox Alpha” and is labeled a preview. It was publicly callable under the canonical whole-scenario request and forced-tool transport, and its date is its OpenRouter listing date. After the row’s run, Z.ai identified the listing as GLM-5.3-Flash (OpenRouter 2026); the row carries the disclosed name and keeps the preview qualifier, because whether the checkpoint that answered is the one later released under that name has not been stated, and it is distinct from the GLM-5.3 row, which is Z.ai’s larger model called under its own id. Its artifact key stays ox-alpha, the name it was queried under.
Frozen snapshot and open-set status
Leaderboard positions are version-sensitive, so manuscript claims refer to the frozen source-run exports in Table 1; the live site can change after them. The committed source-run exports are the manuscript artifacts. The public site exposes the current prompts, predictions, explanations, and reference outputs for transparency. This makes the public leaderboard open-set: models, model providers, or benchmark users could learn from the released cases before later runs. Protected leaderboard claims would require a separate held-out or rotating set.
@@ -206,11 +206,11 @@ Frozen snapshot an
| Snapshot date |
-2026-10-27 |
+2026-11-11 |
| Model response date |
-2026-06-12 to 2026-10-26 |
+2026-06-12 to 2026-11-09 |
| Policy period |
@@ -222,7 +222,7 @@ Frozen snapshot an
| Frozen export SHA-256 prefix |
-US b0db08bc6ec5 |
+US 3b2e92845abb |
| Snapshot manifest |
@@ -250,7 +250,7 @@ Frozen snapshot an
| Scoring code SHA-256 prefix |
-0fdef4505de8 |
+62ca2eb20a84 |
| US run label |
@@ -278,7 +278,7 @@ Frozen snapshot an
| Models |
-51 |
+52 |
| Output groups |
@@ -299,12 +299,12 @@ Frozen snapshot an
-
+| Claude Fable 5.2 |
+92.0 |
+$0.317 |
+48 s |
+
+
| GPT-6 Astra |
91.7 |
$0.126 |
32 s |
-
+
| Claude Sonnet 5.5 |
91.7 |
$0.034 |
20 s |
-
+
| GPT-6 Luna |
91.5 |
$0.002 |
29 s |
-
+
| Claude Fable 5.1 |
90.8 |
$0.257 |
49 s |
-
+
| Kimi K3 |
90.5 |
$0.470 |
1023 s |
-
+
| GPT-6.1 Sol |
90.2 |
$0.023 |
34 s |
-
+
| Muse Wonder |
90.0 |
$0.079 |
26 s |
-
+
| GPT-5.6 Luna |
88.7 |
$0.019 |
14 s |
-
+
| Gemini 4 Argon |
88.6 |
$0.028 |
22 s |
-
+
| GLM-5.3-Flash (preview) |
88.6 |
— |
242 s |
-
+
| GPT-5.5 |
88.0 |
$0.262 |
— |
-
+
| Inkling |
87.7 |
$0.091 |
250 s |
-
+
| Grok 4.7 |
87.6 |
$0.260 |
547 s |
-
+
| GPT-5.6 Terra |
87.6 |
$0.039 |
19 s |
-
+
| Muse Spark 1.3 |
87.2 |
$0.021 |
41 s |
-
+
| DeepSeek V4.1 Flash |
86.3 |
$0.024 |
85 s |
-
+
| Grok 4.6 |
85.8 |
$0.087 |
219 s |
-
+
| Gemini 3.8 Flash |
85.6 |
$0.068 |
56 s |
-
+
| Grok 4.5 |
84.7 |
$0.073 |
112 s |
-
+
| Claude Opus 5 |
83.6 |
$0.067 |
16 s |
-
+
| Claude Fable 5 |
83.3 |
$0.541 |
98 s |
-
+
| DeepSeek V4 Pro 0813 |
83.0 |
$0.199 |
808 s |
-
+
| DeepSeek V4 Flash 0731 |
82.9 |
$0.003 |
195 s |
-
+
| Gemini 3.7 Flash |
82.7 |
$0.014 |
9 s |
-
+
| Gemini 3.6 Flash |
82.6 |
$0.055 |
26 s |
-
+
| Gemini 3.5 Flash-Lite |
81.6 |
$0.003 |
3 s |
-
+
| Gemini 3.1 Pro Preview |
81.2 |
$0.085 |
45 s |
-
+
| GLM-5.3 |
81.1 |
$0.169 |
134 s |
-
+
| Gemini 3 Flash Preview |
80.6 |
$0.048 |
58 s |
-
+
| Claude Opus 4.7 |
80.5 |
$0.293 |
53 s |
-
+
| Grok 4.3 |
80.3 |
$0.015 |
22 s |
-
+
| Claude Sonnet 4.6 |
79.9 |
$0.186 |
114 s |
-
+
| Grok Build 0.1 |
79.9 |
$0.045 |
135 s |
-
+
| Gemini 3.5 Flash |
79.8 |
$0.056 |
24 s |
-
+
| Gemini 3.1 Flash Lite Preview |
79.3 |
$0.002 |
3 s |
-
+
| DeepSeek V4 Pro |
79.3 |
$0.011 |
170 s |
-
+
| Claude Haiku 5.5 |
79.2 |
$0.001 |
5 s |
-
+
| Qwen3.7-max |
76.9 |
$0.139 |
— |
-
+
| GLM-5.2 |
76.5 |
$0.082 |
— |
-
+
| Claude Opus 4.8 |
75.8 |
$0.255 |
55 s |
-
+
| MiniMax M3 |
75.4 |
$0.007 |
— |
-
+
| Qwen3.8-Max |
74.7 |
$0.224 |
861 s |
-
+
| Claude Haiku 4.5 |
74.5 |
$0.055 |
48 s |
-
+
| GPT-5.4 mini |
73.3 |
$0.006 |
5 s |
-
+
| Claude Sonnet 5 |
72.4 |
$0.172 |
64 s |
-
+
| Kimi K2.6 |
66.9 |
$0.391 |
— |
-
+
| GPT-5.4 nano |
65.3 |
$0.002 |
@@ -1557,21 +1573,21 @@ Sensitivity to benchmark view
Bounded score |
GPT-6 Sol (98.1) |
Claude Opus 5.5 (97.9) |
-Muse Wonder (97.5) |
+Claude Fable 5.2 (97.8) |
| 3 |
Equal-output-group score |
Claude Opus 5.5 (98.1) |
+GPT-6 Astra (98.0) |
GPT-6 Sol (98.0) |
-GPT-6 Astra (97.9) |
| 4 |
Amount outputs only |
+Claude Fable 5.2 (97.8) |
Claude Opus 5.5 (97.8) |
-GPT-6 Astra (97.5) |
-GPT-6.1 Sol (97.5) |
+GPT-6 Astra (97.6) |
| 5 |
@@ -1585,7 +1601,7 @@ Sensitivity to benchmark view
Zero reference cases only |
Claude Opus 5.5 (99.5) |
GPT-6 Sol (99.5) |
-GPT-6.1 Terra (99.2) |
+GPT-6.1 Terra (99.3) |
@@ -1596,7 +1612,7 @@ Sensitivity to benchmark view
-
The bounded score is a global-output-weighted average of row scores. Amount outputs use max(0, 1 − |pred − ref| / |ref|) when the reference is nonzero and 1{pred = 0} when the reference is zero. Binary outputs use exact 0/1 matching. The exact-match rate uses the same weighting and aggregation but replaces the amount row score with the headline indicator: 1{|pred − ref| ≤ 1} (one currency unit). Each output group’s default household-impact weight is computed from a full weighting population, the certified PolicyEngine US populace dataset; the 100 benchmark households do not enter it. For each source household, the share is |ref_ij| / max(|household_net_income_i|, Σ_k |ref_ik|); shares are averaged with calibrated household weights and renormalized so output weights sum to one. The max(...) denominator anchors per-household shares to net income when net income is the dominant flow and falls back to the gross tax-benefit flow only when programs cancel each other out, so a $1 benefit to a high-earner contributes essentially zero weight and per-household shares never exceed one. Booleans carry weight through PolicyEngine’s paired per-capita value (for example medicaid_value), so eligibility calls are graded as accuracy but weighted by the dollar stake. Table 10 compares the bounded-score ranking against two opt-in alternatives shown for transparency: equal weighting (every output the same) and budget-weighted (each output’s share of total absolute reference dollars in the same full weighting population).
+The bounded score is a global-output-weighted average of row scores. Amount outputs use max(0, 1 − |pred − ref| / |ref|) when the reference is nonzero and 1{pred = 0} when the reference is zero. Binary outputs use exact 0/1 matching. The exact-match rate uses the same weighting and aggregation but replaces the amount row score with the headline indicator: 1{|pred − ref| ≤ 1} (one currency unit). Each output group’s default household-impact weight is computed from a full weighting population, the certified PolicyEngine US Microcosm dataset; the 100 benchmark households do not enter it. For each source household, the share is |ref_ij| / max(|household_net_income_i|, Σ_k |ref_ik|); shares are averaged with calibrated household weights and renormalized so output weights sum to one. The max(...) denominator anchors per-household shares to net income when net income is the dominant flow and falls back to the gross tax-benefit flow only when programs cancel each other out, so a $1 benefit to a high-earner contributes essentially zero weight and per-household shares never exceed one. Booleans carry weight through PolicyEngine’s paired per-capita value (for example medicaid_value), so eligibility calls are graded as accuracy but weighted by the dollar stake. Table 10 compares the bounded-score ranking against two opt-in alternatives shown for transparency: equal weighting (every output the same) and budget-weighted (each output’s share of total absolute reference dollars in the same full weighting population).