Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 112 additions & 15 deletions scripts/tool_pilot_argon.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,10 @@
# - school_meal_tier is free at or under 130% of poverty, with categorical
# eligibility, in a universal-free-meals state (CA, CO, MA, ME, MI, MN, NM,
# NY and VT in 2026), or where the state pays the reduced-price copay (NC);
# reduced price up to 185%.
# reduced price up to 185%. Both parameters describe breakfast and lunch
# together (state_universal_free_meals: "free breakfast and lunch to all
# students"), so the wording says so: a breakfast-only program is not free
# meals here.
# - is_chip_eligible_child excludes anyone Medicaid-eligible, and chip_category
# includes CHIP coverage of pregnant women.
# - is_medicaid_eligible counts medicaid_category's groups (children, adults,
Expand Down Expand Up @@ -225,9 +228,10 @@
"everyone aged 5 to 17 as attending a public K-12 school that takes part "
"in the National School Lunch and School Breakfast Programs, and count "
"meals at no charge to the family, whether through federal free-meal "
"eligibility or a state program that makes meals free for all students "
"or pays the reduced-price charge (1 if yes, 0 if no; 0 if no one is "
"aged 5 to 17; reduced-price meals do not count as 1)",
"eligibility or a state program that makes both breakfast and lunch free "
"for all students or pays the reduced-price charge for both breakfast and "
"lunch (1 if yes, 0 if no; 0 if no one is aged 5 to 17; reduced-price "
"meals do not count as 1)",
),
(
re.compile(
Expand Down Expand Up @@ -260,6 +264,25 @@
"to the cent, even where the Form 1040 instructions would use the Tax "
"Table",
),
(
# The refundable CTC depends on the tax liability, so the schedule
# choice can move this output too.
re.compile(
r"(total refundable federal income tax credits, including EITC and "
r"refundable portions of credits such as refundable CTC when "
r"applicable; exclude the ACA Premium Tax Credit)"
),
"{who}. Where a credit depends on income tax liability, compute that "
"liability from the section 1 tax rate schedules, not the Tax Table",
),
)

# What a JSON-contract web arm hears once its search-only and calculation turns
# reach the cap. Its API takes no forced answer, so this is the only stop short
# of MAX_TOTAL_TURNS.
JSON_SEARCH_CAP_TEXT = (
"You have used all {cap} tool turns. Reply now with your final answers as "
"the JSON object described above."
)

# Max (2026-10-21): tell the web arms not to look for the answer keys.
Expand Down Expand Up @@ -292,6 +315,18 @@ def v2_pilot_strings(value):
return value


def prompt_wording_sha256(profile: "ModelProfile") -> str | None:
"""A hash of the v2-pilot rewrites a profile sends, so a resume cannot
add households under different wording; None for the board's own v1."""
if profile.prompt_version == "v1":
return None
wording = [
[pattern.pattern, text]
for pattern, text in (*V2_PILOT_DEFINITIONS, *V2_PILOT_CONVENTIONS)
]
return hashlib.sha256(json.dumps(wording).encode()).hexdigest()


def refuse_policyengine(label: str, value) -> None:
"""A v2-pilot guard: no board text the model sees may name PolicyEngine."""
text = value if isinstance(value, str) else json.dumps(value)
Expand All @@ -309,6 +344,8 @@ def refuse_policyengine(label: str, value) -> None:
# for both providers that filter. The Gemini API's search tool takes no domain
# list (it refuses excludeDomains), so its queries are audited instead.
BLOCKED_DOMAINS = ("policybench.org", "github.com", "githubusercontent.com")
# Text that marks a search or result as reaching the reference engine itself.
REFERENCE_ENGINE_MARKERS = ("policyengine",)
# Dynamic filtering: Claude filters results in code before they reach its
# context (platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool).
ANTHROPIC_WEB_SEARCH_TOOL = "web_search_20260209"
Expand Down Expand Up @@ -1552,6 +1589,7 @@ def turn_searches(
"kind": "source",
"id": _field(item, "id"),
"url": _field(source, "url"),
"source": _plain(source),
}
)
return log, billed
Expand Down Expand Up @@ -1722,6 +1760,7 @@ def run_household(
"variables": variables,
"board_prompt_sha256": board_prompt_sha256,
"prompt_version": profile.prompt_version,
"prompt_wording_sha256": prompt_wording_sha256(profile),
"benchmark_search_rule": profile.benchmark_search_rule,
"sent_board_prompt_sha256": sent_board_prompt_sha256,
"tool_notice_sha256": hashlib.sha256(profile.notice.encode()).hexdigest(),
Expand Down Expand Up @@ -1753,6 +1792,19 @@ def tools_and_choice(kind: str):
return [*offered, answer_tool], forced_answer_choice(profile)
return [*offered, answer_tool], "required"

def cap_reached() -> bool:
return tool_turns + search_only_turns >= turn_cap

def count_search_only(turn_record: dict, turn) -> bool:
"""Count a web arm's turn that searched and neither answered nor
calculated toward the cap, so a model cannot search without end."""
nonlocal search_only_turns
if profile.uses_web and (turn.server_calls or turn_record.get("web_searches")):
search_only_turns += 1
turn_record["search_only_turn"] = True
return True
return False

def bill_searches(response, turn) -> dict:
"""Log the response's searches and add their price to the spend."""
if not profile.uses_web:
Expand Down Expand Up @@ -2048,13 +2100,14 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool:
answered = any(v is not None for v in predictions.values()) or any(
explanations.values()
)
for call in other_calls:
conversation.add_tool_result(call, {"error": "unknown function"})
if not answered:
for call in other_calls:
conversation.add_tool_result(
call, {"error": "unknown function"}
)
if turn.unfinished_server_calls:
continue # only tool results may follow
if count_search_only(turn_record, turn) and cap_reached():
conversation.add_user(JSON_SEARCH_CAP_TEXT.format(cap=turn_cap))
continue
conversation.add_user(
"Reply with your final answers as the JSON object described "
"above"
Expand All @@ -2068,13 +2121,12 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool:
else:
for call in other_calls:
conversation.add_tool_result(call, {"error": "unknown function"})
if profile.uses_web and (
turn.server_calls or turn_record.get("web_searches")
):
search_only_turns += 1
turn_record["search_only_turn"] = True
searched_only = count_search_only(turn_record, turn)
if turn.unfinished_server_calls:
continue # only tool results may follow
if json_contract and searched_only and cap_reached():
conversation.add_user(JSON_SEARCH_CAP_TEXT.format(cap=turn_cap))
continue
if calculate_offered:
nudge = (
"Call calculate_household or give your final answers as "
Expand All @@ -2097,8 +2149,14 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool:
if repair_rounds >= MAX_REPAIR_ROUNDS:
record["status"] = "incomplete"
break
repair_rounds += 1
answer_variables = still
if turn.unfinished_server_calls:
# Only tool results may follow an unfinished server call. The
# next reply is merged for the outputs listed here; no repair
# round is used, since no repair text was sent.
turn_record["repair_deferred"] = True
continue
repair_rounds += 1
conversation.add_user(repair_text(still))
else:
record["status"] = "turn_limit"
Expand Down Expand Up @@ -2384,6 +2442,22 @@ def records_prompt(records: Mapping[str, dict]) -> tuple[str, bool]:
return setups.pop() if setups else ("v1", False)


def records_wording(records: Mapping[str, dict]) -> str | None:
"""The one v2-pilot wording hash a run directory's records share. v2-pilot
records written before the hash existed read as "unrecorded", so a resume
into them is refused rather than mixed."""
hashes = {
r.get(
"prompt_wording_sha256",
"unrecorded" if r.get("prompt_version", "v1") != "v1" else None,
)
for r in records.values()
}
if len(hashes) > 1:
raise SystemExit(f"records under more than one prompt wording: {hashes}")
return hashes.pop() if hashes else None


def records_arm(records: Mapping[str, dict]) -> str:
"""The one arm a run directory's records belong to; records written
before the web arms are the PE tool's."""
Expand Down Expand Up @@ -2434,6 +2508,11 @@ def command_run(args) -> None:
f"{households_dir} holds records with prompt setup "
f"{records_prompt(previous)}, not {setup}"
)
if previous and records_wording(previous) != prompt_wording_sha256(profile):
raise SystemExit(
f"{households_dir} holds records sent under other v2-pilot wording "
f"({records_wording(previous)}); use a fresh --out-dir"
)
spend = Spend(args.budget_usd, spent_before)
todo = [sid for sid in ids if sid not in done]
manifest = {
Expand All @@ -2448,6 +2527,7 @@ def command_run(args) -> None:
"model_id": profile.model_id,
"arm": profile.arm,
"prompt_version": profile.prompt_version,
"prompt_wording_sha256": prompt_wording_sha256(profile),
"benchmark_search_rule": profile.benchmark_search_rule,
"v2_pilot_definitions": [
{"v1": pattern.pattern, "v2_pilot": text}
Expand Down Expand Up @@ -2727,6 +2807,11 @@ def command_score(args) -> None:
if profile.uses_web:
searches.to_csv(out / "searches.csv", index=False)
audit = search_audit(searches, results_visible=profile.provider != "gemini")
if not audit["results_visible"]:
audit["caveat"] = (
"This provider returns no result URLs, so flags and the unflagged "
"score rest on queries alone; what the model read cannot be audited."
)
clean = [sid for sid in wide.index if sid not in audit["flagged_households"]]
audit["weighted_exact_unflagged"] = {
"households": len(clean),
Expand All @@ -2738,8 +2823,11 @@ def command_score(args) -> None:
f"{len(audit['policybench_mentions'])}; blocked-domain URLs: "
f"{len(audit['blocked_domain_urls'])}; flagged households: "
f"{audit['flagged_households']}; results visible: "
f"{audit['results_visible']}"
f"{audit['results_visible']}; entries naming the reference engine: "
f"{audit['reference_engine_entries']}"
)
if not audit["results_visible"]:
print(f"CAVEAT: {audit['caveat']}")
(out / "score.json").write_text(
json.dumps(
{
Expand Down Expand Up @@ -2976,8 +3064,17 @@ def records(frame: pd.DataFrame) -> list[dict]:
# JSON has no NaN: missing cells become null.
return frame.astype(object).where(frame.notna(), None).to_dict(orient="records")

# The reference engine's own sites: not blocked and not a flag, since a
# web+PE model may read the tool's docs, but counted for the reader.
engine = log.loc[
pd.array(
[touches(t, REFERENCE_ENGINE_MARKERS) for t in text],
dtype="boolean",
)
]
return {
"results_visible": results_visible,
"reference_engine_entries": int(len(engine)),
"logged_entries": int(len(log)),
"queries": int(log["query"].notna().sum()),
"result_urls": int(log["url"].notna().sum()),
Expand Down
Loading
Loading