From 7210d03c0f39185b4c0d6a4cac3791d05db4df6c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 21 Oct 2026 08:38:06 -0400 Subject: [PATCH 1/2] Close review r2's follow-ups on the tool pilot's web arms - v2-pilot meal wording names breakfast and lunch, as the reference's parameters do (state_universal_free_meals: "free breakfast and lunch to all students"; the copay parameter covers both meals). A breakfast-only program is not free meals here. - The d1263 rate-schedule line now rides on federal_refundable_credits too, since the refundable CTC depends on tax liability. - JSON-contract web arms count search-only turns toward the cap, and hear a cap message once it is reached. Before, the cap bound only tool-contract models. - An answer beside an unknown call gets the call's result. A repair waits while a server call is unfinished, because text there is a 400. - The audit keeps each OpenAI source object whole, counts entries that name the reference engine, and prints a caveat when a provider returns no result URLs (Gemini). - Tests pin litellm's block order when a deferred server result is moved next to its call. That case has thinking blocks and a nested (caller) search, as real Opus replies do. Two live probes with the board's settings ran four requests each, clean; their full request and response bodies are saved in the pilot record. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 88 ++++++++++--- tests/test_tool_pilot_argon.py | 222 ++++++++++++++++++++++++++++++++- 2 files changed, 291 insertions(+), 19 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 1a68322c..9659919f 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -184,7 +184,10 @@ # - school_meal_tier is free at or under 130% of poverty, with categorical # eligibility, in a universal-free-meals state (CA, CO, MA, ME, MI, MN, NM, # NY and VT in 2026), or where the state pays the reduced-price copay (NC); -# reduced price up to 185%. +# reduced price up to 185%. Both parameters describe breakfast and lunch +# together (state_universal_free_meals: "free breakfast and lunch to all +# students"), so the wording says so: a breakfast-only program is not free +# meals here. # - is_chip_eligible_child excludes anyone Medicaid-eligible, and chip_category # includes CHIP coverage of pregnant women. # - is_medicaid_eligible counts medicaid_category's groups (children, adults, @@ -225,9 +228,10 @@ "everyone aged 5 to 17 as attending a public K-12 school that takes part " "in the National School Lunch and School Breakfast Programs, and count " "meals at no charge to the family, whether through federal free-meal " - "eligibility or a state program that makes meals free for all students " - "or pays the reduced-price charge (1 if yes, 0 if no; 0 if no one is " - "aged 5 to 17; reduced-price meals do not count as 1)", + "eligibility or a state program that makes both breakfast and lunch free " + "for all students or pays the reduced-price charge for both breakfast and " + "lunch (1 if yes, 0 if no; 0 if no one is aged 5 to 17; reduced-price " + "meals do not count as 1)", ), ( re.compile( @@ -260,6 +264,25 @@ "to the cent, even where the Form 1040 instructions would use the Tax " "Table", ), + ( + # The refundable CTC depends on the tax liability, so the schedule + # choice can move this output too. + re.compile( + r"(total refundable federal income tax credits, including EITC and " + r"refundable portions of credits such as refundable CTC when " + r"applicable; exclude the ACA Premium Tax Credit)" + ), + "{who}. Where a credit depends on income tax liability, compute that " + "liability from the section 1 tax rate schedules, not the Tax Table", + ), +) + +# What a JSON-contract web arm hears once its search-only and calculation turns +# reach the cap. Its API takes no forced answer, so this is the only stop short +# of MAX_TOTAL_TURNS. +JSON_SEARCH_CAP_TEXT = ( + "You have used all {cap} tool turns. Reply now with your final answers as " + "the JSON object described above." ) # Max (2026-10-21): tell the web arms not to look for the answer keys. @@ -309,6 +332,8 @@ def refuse_policyengine(label: str, value) -> None: # for both providers that filter. The Gemini API's search tool takes no domain # list (it refuses excludeDomains), so its queries are audited instead. BLOCKED_DOMAINS = ("policybench.org", "github.com", "githubusercontent.com") +# Text that marks a search or result as reaching the reference engine itself. +REFERENCE_ENGINE_MARKERS = ("policyengine",) # Dynamic filtering: Claude filters results in code before they reach its # context (platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool). ANTHROPIC_WEB_SEARCH_TOOL = "web_search_20260209" @@ -1552,6 +1577,7 @@ def turn_searches( "kind": "source", "id": _field(item, "id"), "url": _field(source, "url"), + "source": _plain(source), } ) return log, billed @@ -1753,6 +1779,19 @@ def tools_and_choice(kind: str): return [*offered, answer_tool], forced_answer_choice(profile) return [*offered, answer_tool], "required" + def cap_reached() -> bool: + return tool_turns + search_only_turns >= turn_cap + + def count_search_only(turn_record: dict, turn) -> bool: + """Count a web arm's turn that searched and neither answered nor + calculated toward the cap, so a model cannot search without end.""" + nonlocal search_only_turns + if profile.uses_web and (turn.server_calls or turn_record.get("web_searches")): + search_only_turns += 1 + turn_record["search_only_turn"] = True + return True + return False + def bill_searches(response, turn) -> dict: """Log the response's searches and add their price to the spend.""" if not profile.uses_web: @@ -2048,13 +2087,14 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool: answered = any(v is not None for v in predictions.values()) or any( explanations.values() ) + for call in other_calls: + conversation.add_tool_result(call, {"error": "unknown function"}) if not answered: - for call in other_calls: - conversation.add_tool_result( - call, {"error": "unknown function"} - ) if turn.unfinished_server_calls: continue # only tool results may follow + if count_search_only(turn_record, turn) and cap_reached(): + conversation.add_user(JSON_SEARCH_CAP_TEXT.format(cap=turn_cap)) + continue conversation.add_user( "Reply with your final answers as the JSON object described " "above" @@ -2068,13 +2108,12 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool: else: for call in other_calls: conversation.add_tool_result(call, {"error": "unknown function"}) - if profile.uses_web and ( - turn.server_calls or turn_record.get("web_searches") - ): - search_only_turns += 1 - turn_record["search_only_turn"] = True + searched_only = count_search_only(turn_record, turn) if turn.unfinished_server_calls: continue # only tool results may follow + if json_contract and searched_only and cap_reached(): + conversation.add_user(JSON_SEARCH_CAP_TEXT.format(cap=turn_cap)) + continue if calculate_offered: nudge = ( "Call calculate_household or give your final answers as " @@ -2099,6 +2138,11 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool: break repair_rounds += 1 answer_variables = still + if turn.unfinished_server_calls: + # Only tool results may follow an unfinished server call; the + # next reply is still merged for the outputs listed here. + turn_record["repair_deferred"] = True + continue conversation.add_user(repair_text(still)) else: record["status"] = "turn_limit" @@ -2738,8 +2782,15 @@ def command_score(args) -> None: f"{len(audit['policybench_mentions'])}; blocked-domain URLs: " f"{len(audit['blocked_domain_urls'])}; flagged households: " f"{audit['flagged_households']}; results visible: " - f"{audit['results_visible']}" + f"{audit['results_visible']}; entries naming the reference engine: " + f"{audit['reference_engine_entries']}" ) + if not audit["results_visible"]: + print( + "CAVEAT: this provider returns no result URLs, so flags and the " + "unflagged score rest on queries alone; what the model read " + "cannot be audited." + ) (out / "score.json").write_text( json.dumps( { @@ -2976,8 +3027,17 @@ def records(frame: pd.DataFrame) -> list[dict]: # JSON has no NaN: missing cells become null. return frame.astype(object).where(frame.notna(), None).to_dict(orient="records") + # The reference engine's own sites: not blocked and not a flag, since a + # web+PE model may read the tool's docs, but counted for the reader. + engine = log.loc[ + pd.array( + [touches(t, REFERENCE_ENGINE_MARKERS) for t in text], + dtype="boolean", + ) + ] return { "results_visible": results_visible, + "reference_engine_entries": int(len(engine)), "logged_entries": int(len(log)), "queries": int(log["query"].notna().sum()), "result_urls": int(log["url"].notna().sum()), diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index 50ebb062..c5734d88 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -3082,12 +3082,20 @@ def test_the_v2_pilot_prompt_names_policyengine_nowhere_in_any_household(): old_lines, new_lines = old.splitlines(), new.splitlines() assert len(old_lines) == len(new_lines) changed = [(a, b) for a, b in zip(old_lines, new_lines) if a != b] - federal = "- federal_income_tax_before_refundable_credits:" + federal = ( + "- federal_income_tax_before_refundable_credits:", + "- federal_refundable_credits:", + ) assert all("PolicyEngine" in a or a.startswith(federal) for a, _ in changed) - assert len(changed) == old.count("PolicyEngine") + (federal in old) - # d1263: the v2 prompt states the rate-schedule convention. - if federal in old: - assert "section 1 tax rate schedules" in new + assert len(changed) == old.count("PolicyEngine") + sum( + line in old for line in federal + ) + # d1263: the v2 prompt states the rate-schedule convention on both + # federal lines whose values the schedule choice can move. + for line in federal: + for old_line, new_line in zip(old_lines, new_lines): + if old_line.startswith(line): + assert "section 1 tax rate schedules" in new_line def test_v2_pilot_keeps_each_persons_label_in_their_definition(): @@ -3192,3 +3200,207 @@ def test_records_name_one_prompt_setup_and_old_records_are_v1(): ) == ("v2-pilot", True) with pytest.raises(SystemExit): pilot.records_prompt({"a": {"prompt_version": "v2-pilot"}, "b": {}}) + + +# --------------------------------------------------------------------------- +# Review r2 follow-ups. +# --------------------------------------------------------------------------- + + +def finished_search(call_id="srvtoolu_s", content=None): + """A Claude turn whose search ran to its result in the same response.""" + return with_fields( + response([tool_call("web_search", {"query": "q"}, call_id)], content=content), + { + "web_search_results": [ + {"type": "web_search_tool_result", "tool_use_id": call_id} + ] + }, + server_tool_use=1, + ) + + +def test_a_json_web_arm_hears_the_cap_after_search_only_turns(): + cap = 2 + model = FakeModel( + [finished_search(f"srvtoolu_{i}", content="Still looking.") for i in range(cap)] + + [response([], content=json_answer({"snap": 1, VARIABLES[1]: 2}))] + ) + record = pilot.run_household( + make_scenario(), + VARIABLES, + fake_tool, + model, + pilot.Spend(50.0), + turn_cap=cap, + profile=pilot.model_profile(OPUS, "web"), + ) + assert record["status"] == "complete" and record["search_only_turns"] == cap + first_nudge = model.requests[1]["messages"][-1]["content"][0]["text"] + cap_text = model.requests[cap]["messages"][-1]["content"][0]["text"] + assert "used all" not in first_nudge + assert cap_text == pilot.JSON_SEARCH_CAP_TEXT.format(cap=cap) + + +def test_an_answer_beside_an_unknown_call_gets_its_result(): + model = FakeModel( + [ + response( + [tool_call("lookup", {"q": 1}, "u1")], + content=json_answer({"snap": 1}), + ), + response([], content=json_answer({VARIABLES[1]: 2})), + ] + ) + record = run_opus(model) + assert record["status"] == "complete" and record["repair_rounds"] == 1 + messages = model.requests[1]["messages"] + assert messages[-2]["role"] == "tool" and messages[-2]["tool_call_id"] == "u1" + assert messages[-1]["role"] == "user" + + +def test_a_repair_waits_while_a_server_call_is_unfinished(): + """A partial answer beside an unknown call and a deferred server call: + the unknown call gets its result, and no repair text may follow yet.""" + deferred = response( + [ + tool_call("web_search", {"query": "q"}, "srvtoolu_d"), + tool_call("lookup", {"q": 1}, "u1"), + ], + content=json_answer({"snap": 1}), + ) + resumed = with_fields( + response([], content=json_answer({VARIABLES[1]: 2})), + { + "web_search_results": [ + {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_d"} + ] + }, + ) + model = FakeModel([deferred, resumed]) + record = pilot.run_household( + make_scenario(), + VARIABLES, + fake_tool, + model, + pilot.Spend(50.0), + profile=pilot.model_profile(OPUS, "web"), + ) + assert record["status"] == "complete" and record["repair_rounds"] == 1 + assert record["turns"][0]["repair_deferred"] is True + last = model.requests[1]["messages"][-1] + assert last["role"] == "tool" and last["tool_call_id"] == "u1" + assert record["predictions"] == {"snap": 1.0, VARIABLES[1]: 2.0} + + +def test_relocation_keeps_litellms_block_order_with_thinking_and_nested_searches(): + """Real Opus replies carry thinking blocks, so litellm rebuilds them in + its interleaved order; the deferred result must land after its call.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + anthropic_messages_pt, + ) + + thinking = [{"type": "thinking", "thinking": "plan", "signature": "sig1"}] + first = response( + [tool_call("code_execution", {"code": "x"}, "srvtoolu_a"), calculate("c1")] + ) + first.choices[0].message.thinking_blocks = thinking + nested = {"type": "code_execution_20260120", "tool_id": "srvtoolu_a"} + second = with_fields( + response( + [ + tool_call("web_search", {"query": "q"}, "srvtoolu_b"), + calculate("c2"), + ] + ), + { + "tool_results": [ + { + "type": "code_execution_tool_result", + "tool_use_id": "srvtoolu_a", + "content": {"type": "code_execution_result", "stdout": "ok"}, + } + ], + "web_search_results": [ + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_b", + "caller": nested, + "content": [], + } + ], + }, + ) + second.choices[0].message.thinking_blocks = [ + {"type": "thinking", "thinking": "more", "signature": "sig2"} + ] + model = FakeModel( + [first, second, response([], content=json_answer({"snap": 1, VARIABLES[1]: 2}))] + ) + record = pilot.run_household( + make_scenario(), + VARIABLES, + fake_tool, + model, + pilot.Spend(50.0), + profile=pilot.model_profile(OPUS, "web+pe"), + ) + assert record["status"] == "complete" + converted = anthropic_messages_pt( + as_data(record["messages"]), model="claude-opus-5-5", llm_provider="anthropic" + ) + replies = [m["content"] for m in converted if m["role"] == "assistant"] + first_kinds = [(b["type"], b.get("id") or b.get("tool_use_id")) for b in replies[0]] + assert first_kinds[0][0] == "thinking" + a_call = first_kinds.index(("server_tool_use", "srvtoolu_a")) + assert first_kinds[a_call + 1] == ("code_execution_tool_result", "srvtoolu_a") + assert ("tool_use", "c1") in first_kinds + second_kinds = [ + (b["type"], b.get("id") or b.get("tool_use_id")) for b in replies[1] + ] + assert ("code_execution_tool_result", "srvtoolu_a") not in second_kinds + b_call = second_kinds.index(("server_tool_use", "srvtoolu_b")) + assert second_kinds[b_call + 1] == ("web_search_tool_result", "srvtoolu_b") + + +def test_the_meal_wording_names_breakfast_and_lunch(): + text = pilot.v2_pilot_text( + "- free_school_meals_eligible: whether PolicyEngine returns positive annual " + "free school meal support for the household (1 if yes, 0 if no; " + "reduced-price meals do not count as 1)" + ) + assert "both breakfast and lunch free for all students" in text + assert "reduced-price charge for both breakfast and lunch" in text + + +def test_the_audit_keeps_source_objects_and_counts_reference_engine_entries(): + action = SimpleNamespace( + type="search", + query="q", + sources=[SimpleNamespace(type="url", url="https://policyengine.org/us")], + ) + model = FakeResponses( + [ + responses_response( + [ + web_search_call("ws_1", action), + sol_submit({"snap": 1, VARIABLES[1]: 2}), + ], + "r", + ) + ] + ) + record = pilot.run_household( + make_scenario(), + VARIABLES, + fake_tool, + model, + pilot.Spend(50.0), + profile=pilot.model_profile(SOL, "web"), + ) + (source,) = [e for e in record["turns"][0]["searches"] if e["kind"] == "source"] + assert source["source"] == {"type": "url", "url": "https://policyengine.org/us"} + record["turns"][0]["index"] = 0 + audit = pilot.search_audit(pilot.search_log({"s": record})) + assert audit["reference_engine_entries"] == 1 + assert audit["flagged_households"] == [] From 9249fc02c3131b7bb64add449b8bab76c5b501ef Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 21 Oct 2026 08:46:16 -0400 Subject: [PATCH 2/2] Refuse a resume across v2-pilot wording changes; keep deferred repairs free Review r3 of #236 found three issues. - "v2-pilot" named two wordings, since the meal and refundable-credit lines changed after the v2 trial, so a resume could mix them. Each record and manifest now carries prompt_wording_sha256, a hash of every v2-pilot rewrite, and run refuses to resume across a change. Older v2-pilot records read as "unrecorded", so a resume into trial-v2b is refused. - A repair deferred behind an unfinished server call used up a repair round without telling the model what was missing. Now it uses none. - The no-result-URLs caveat is written to score.json's search_audit, not only printed. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 53 +++++++++++++++++++++++++++++----- tests/test_tool_pilot_argon.py | 33 ++++++++++++++++++++- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 9659919f..275dda78 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -315,6 +315,18 @@ def v2_pilot_strings(value): return value +def prompt_wording_sha256(profile: "ModelProfile") -> str | None: + """A hash of the v2-pilot rewrites a profile sends, so a resume cannot + add households under different wording; None for the board's own v1.""" + if profile.prompt_version == "v1": + return None + wording = [ + [pattern.pattern, text] + for pattern, text in (*V2_PILOT_DEFINITIONS, *V2_PILOT_CONVENTIONS) + ] + return hashlib.sha256(json.dumps(wording).encode()).hexdigest() + + def refuse_policyengine(label: str, value) -> None: """A v2-pilot guard: no board text the model sees may name PolicyEngine.""" text = value if isinstance(value, str) else json.dumps(value) @@ -1748,6 +1760,7 @@ def run_household( "variables": variables, "board_prompt_sha256": board_prompt_sha256, "prompt_version": profile.prompt_version, + "prompt_wording_sha256": prompt_wording_sha256(profile), "benchmark_search_rule": profile.benchmark_search_rule, "sent_board_prompt_sha256": sent_board_prompt_sha256, "tool_notice_sha256": hashlib.sha256(profile.notice.encode()).hexdigest(), @@ -2136,13 +2149,14 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool: if repair_rounds >= MAX_REPAIR_ROUNDS: record["status"] = "incomplete" break - repair_rounds += 1 answer_variables = still if turn.unfinished_server_calls: - # Only tool results may follow an unfinished server call; the - # next reply is still merged for the outputs listed here. + # Only tool results may follow an unfinished server call. The + # next reply is merged for the outputs listed here; no repair + # round is used, since no repair text was sent. turn_record["repair_deferred"] = True continue + repair_rounds += 1 conversation.add_user(repair_text(still)) else: record["status"] = "turn_limit" @@ -2428,6 +2442,22 @@ def records_prompt(records: Mapping[str, dict]) -> tuple[str, bool]: return setups.pop() if setups else ("v1", False) +def records_wording(records: Mapping[str, dict]) -> str | None: + """The one v2-pilot wording hash a run directory's records share. v2-pilot + records written before the hash existed read as "unrecorded", so a resume + into them is refused rather than mixed.""" + hashes = { + r.get( + "prompt_wording_sha256", + "unrecorded" if r.get("prompt_version", "v1") != "v1" else None, + ) + for r in records.values() + } + if len(hashes) > 1: + raise SystemExit(f"records under more than one prompt wording: {hashes}") + return hashes.pop() if hashes else None + + def records_arm(records: Mapping[str, dict]) -> str: """The one arm a run directory's records belong to; records written before the web arms are the PE tool's.""" @@ -2478,6 +2508,11 @@ def command_run(args) -> None: f"{households_dir} holds records with prompt setup " f"{records_prompt(previous)}, not {setup}" ) + if previous and records_wording(previous) != prompt_wording_sha256(profile): + raise SystemExit( + f"{households_dir} holds records sent under other v2-pilot wording " + f"({records_wording(previous)}); use a fresh --out-dir" + ) spend = Spend(args.budget_usd, spent_before) todo = [sid for sid in ids if sid not in done] manifest = { @@ -2492,6 +2527,7 @@ def command_run(args) -> None: "model_id": profile.model_id, "arm": profile.arm, "prompt_version": profile.prompt_version, + "prompt_wording_sha256": prompt_wording_sha256(profile), "benchmark_search_rule": profile.benchmark_search_rule, "v2_pilot_definitions": [ {"v1": pattern.pattern, "v2_pilot": text} @@ -2771,6 +2807,11 @@ def command_score(args) -> None: if profile.uses_web: searches.to_csv(out / "searches.csv", index=False) audit = search_audit(searches, results_visible=profile.provider != "gemini") + if not audit["results_visible"]: + audit["caveat"] = ( + "This provider returns no result URLs, so flags and the unflagged " + "score rest on queries alone; what the model read cannot be audited." + ) clean = [sid for sid in wide.index if sid not in audit["flagged_households"]] audit["weighted_exact_unflagged"] = { "households": len(clean), @@ -2786,11 +2827,7 @@ def command_score(args) -> None: f"{audit['reference_engine_entries']}" ) if not audit["results_visible"]: - print( - "CAVEAT: this provider returns no result URLs, so flags and the " - "unflagged score rest on queries alone; what the model read " - "cannot be audited." - ) + print(f"CAVEAT: {audit['caveat']}") (out / "score.json").write_text( json.dumps( { diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index c5734d88..80d455c7 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -3286,7 +3286,8 @@ def test_a_repair_waits_while_a_server_call_is_unfinished(): pilot.Spend(50.0), profile=pilot.model_profile(OPUS, "web"), ) - assert record["status"] == "complete" and record["repair_rounds"] == 1 + # The deferral sent no repair text, so it used no repair round. + assert record["status"] == "complete" and record["repair_rounds"] == 0 assert record["turns"][0]["repair_deferred"] is True last = model.requests[1]["messages"][-1] assert last["role"] == "tool" and last["tool_call_id"] == "u1" @@ -3404,3 +3405,33 @@ def test_the_audit_keeps_source_objects_and_counts_reference_engine_entries(): audit = pilot.search_audit(pilot.search_log({"s": record})) assert audit["reference_engine_entries"] == 1 assert audit["flagged_households"] == [] + + +def test_a_resume_refuses_households_under_other_v2_pilot_wording(monkeypatch): + v2 = pilot.model_profile(SOL, "web", "v2-pilot", True) + current = pilot.prompt_wording_sha256(v2) + assert pilot.prompt_wording_sha256(pilot.model_profile(SOL, "web")) is None + assert current and len(current) == 64 + # Records carry the hash; records from before it read as unrecorded. + assert pilot.records_wording({"a": {"prompt_wording_sha256": current}}) == current + assert pilot.records_wording({"a": {"prompt_version": "v2-pilot"}}) == "unrecorded" + assert pilot.records_wording({"a": {}}) is None + with pytest.raises(SystemExit): + pilot.records_wording( + { + "a": {"prompt_wording_sha256": current}, + "b": {"prompt_version": "v2-pilot"}, + } + ) + # Any change to the rewrites changes the hash. + monkeypatch.setattr(pilot, "V2_PILOT_CONVENTIONS", ()) + assert pilot.prompt_wording_sha256(v2) != current + + +def test_a_run_record_carries_its_wording_hash(): + model = FakeModel([response([submit({"snap": 1, VARIABLES[1]: 2})])]) + profile = pilot.model_profile(pilot.MODEL_NAME, "pe", "v2-pilot") + record = pilot.run_household( + make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile + ) + assert record["prompt_wording_sha256"] == pilot.prompt_wording_sha256(profile)