diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1049d7bd..2ea21be2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -68,6 +68,7 @@ jobs: tests/test_refusal_coverage_citation_example.py \ tests/test_refusal_question_scoped_coverage.py \ tests/test_refusal_v5_3_preparation.py \ + tests/test_refusal_v5_3_runner.py \ tests/test_refusal_v5_2_1_preparation.py \ tests/test_refusal_v5_2_1_runner.py \ tests/test_refusal_natural_diagnostic_preparation.py \ diff --git a/experiments/evals/refusal_v5_3_runner.py b/experiments/evals/refusal_v5_3_runner.py new file mode 100644 index 00000000..22b0bb97 --- /dev/null +++ b/experiments/evals/refusal_v5_3_runner.py @@ -0,0 +1,132 @@ +"""Authorized observed-TRAIN regression; no independent acceptance or serving.""" + +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + +from experiments.evals import refusal_natural_diagnostic_runner as previous +from experiments.evals import refusal_v5_3_preparation as prep + +canonical, paired = prep.canonical, prep.prior.paired +CONTRACT = prep.prior.prior.REPORT / "v5_3_regression_run_contract.json" +RESULT = prep.prior.prior.REPORT / "v5_3_regression_result.json" +ROOT = Path("data/refusal_natural_v5_3/run_v1") +PREPARATION_SHA256 = "403bedd5717d31d1a7561051da2841403ab220503532ec20052a5727b7c2dd7a" + +# ponytail: isolate the existing runner; adapt its preparation names, not its durable engine. +_spec = importlib.util.spec_from_file_location("_v5_3_natural_regression", previous.__file__) +_reuse = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_reuse) +_reuse.prep = SimpleNamespace(**vars(prep), paired=paired, natural=prep.prior.natural, + INPUTS=prep.prior.INPUTS, TARGETS=prep.prior.TARGETS) +_reuse.PREPARATION_SHA256, _reuse.CONTRACT = PREPARATION_SHA256, CONTRACT + + +def execution_contract(manifest): + budget = manifest["preflight"] + return {**manifest, "status": "AUTHORIZED_NOT_RUN", "preparation_canonical_sha256": canonical(manifest), + "runner_normalized_source_sha256": hashlib.sha256(Path(__file__).read_text(encoding="utf-8").encode()).hexdigest(), + "reused_runner_normalized_sha256": hashlib.sha256(Path(previous.__file__).read_text(encoding="utf-8").encode()).hexdigest(), + "max_calls_total": 60, "classifier": budget["classifier"], "pricing_snapshot": budget["pricing_snapshot"], + "arms": {arm: {"max_calls": 30, "max_output_tokens": value["max_output_tokens"], + "request_input_byte_bounds": value["input_byte_plus_framing_bounds"]} + for arm, value in budget["arms"].items()}, + "controls": {**manifest["controls"], "executable_run_contract": True, "paid_runner_ready": True, + "failed_pair_is_terminal": True, "max_failed_attempts_per_arm": 30, "sdk_retries": 0}, + "authorization": "User explicitly confirmed PR77 merged and authorized execution under PR77: 30 questions/full Top14, <=60 Singapore qwen3.5-plus-2026-04-20 calls/CNY8; no labels, DEV, retries, recharge or promotion"} + + +_reuse.execution_contract = execution_contract +prepare_contract = _reuse.prepare_contract +population_metrics = _reuse.population_metrics + + +def load_probe(): + manifest = paired.shared._read_json(prep.REPORT) + if hashlib.sha256(prep.PROTOCOL.read_text(encoding="utf-8").encode()).hexdigest() != manifest["protocol_normalized_sha256"]: + raise RuntimeError("frozen protocol changed") + return _reuse.load_probe() # Input/source/request/contract bindings; never target rows. + + +def parse_result(arm, raw, row, finish_reason): + if arm not in prep.BUILDERS or finish_reason != "stop": + raise ValueError("invalid arm or incomplete response") + return prep.candidate.validate_requirement_coverage_response(raw, paired.shared._classifier_input_from_row(row)) + + +_reuse._engine.BUILDERS, _reuse._engine.parse_result = prep.BUILDERS, parse_result + + +def run_paid(*, root=ROOT, client=None, max_new_calls=60, probe_loader=load_probe): + return _reuse.run_paid(root=root, client=client, max_new_calls=max_new_calls, probe_loader=probe_loader) + + +def evaluate(*, root=ROOT): + contract, rows = load_probe() + events, completed, _, spent = paired.load_state(root, rows, contract, builders=prep.BUILDERS, + result_parser=parse_result, contract_sha256=canonical(contract)) + if completed != 60 or (Path(root) / "runner.lock").exists(): + raise RuntimeError("all 60 terminal outcomes and stopped runner required") + if paired.shared._sha256(prep.prior.TARGETS) != contract["targets_sha256"]: + raise RuntimeError("frozen target hash changed") + labels = paired.shared._read_jsonl(prep.prior.TARGETS) # After all paid outcomes, never in classifier requests. + if [t["question_id"] for t in labels] != contract["case_order"]: + raise RuntimeError("target order changed") + outcomes = {(e["question_id"], e["arm"]): e["result"] for e in events} + transitions = [{"question_id": t["question_id"], "target_class": t["target_class"], + **{arm: outcomes[t["question_id"], arm]["decision"] if outcomes[t["question_id"], arm] else None + for arm in prep.BUILDERS}} for t in labels] + primary = population_metrics(transitions) + if primary["target_counts"] != contract["target_counts"]: + raise RuntimeError("frozen target counts changed") + common = [t for t in transitions if t["target_class"] != "AMBIGUOUS" and all(t[a] is not None for a in prep.BUILDERS)] + summary = {"status": "OBSERVED_DEVELOPMENT_REGRESSION_COMPLETE_NO_ACCEPTANCE", "provider_calls": len(events), + "estimated_cost_cny": spent, "primary": primary, "common_valid_binary": population_metrics(common), + "all_outcome_transitions": {f"{a or 'FAILURE'}_to_{b or 'FAILURE'}": sum( + t["v5_2_1"] == a and t["v5_3"] == b for t in transitions) + for a in ("SUFFICIENT", "INSUFFICIENT", None) for b in ("SUFFICIENT", "INSUFFICIENT", None)}, + "paired_binary_agreement_transitions": {name: sum((t["v5_2_1"] == t["target_class"], t["v5_3"] == t["target_class"]) == state for t in common) + for name, state in (("both_agree", (True, True)), ("baseline_only", (True, False)), + ("candidate_only", (False, True)), ("neither_agrees", (False, False)))}, + "family_sensitivity": {f"drop_{qid}": population_metrics([t for t in transitions if t["question_id"] != qid]) + for qid in contract["known_correlated_pair"]}, + "usage": {arm: {"calls": sum(e["arm"] == arm for e in events), + "prompt_tokens": sum(e["prompt_tokens"] for e in events if e["arm"] == arm), + "completion_tokens": sum(e["completion_tokens"] for e in events if e["arm"] == arm), + "estimated_cost_cny": sum(e["estimated_cost_cny"] for e in events if e["arm"] == arm), + "latency_p50_ms": paired.shared._percentile([e["latency_ms"] for e in events if e["arm"] == arm], .5)} + for arm in prep.BUILDERS}, + "contract_canonical_sha256": canonical(contract), "targets_sha256": contract["targets_sha256"], + "journal_sha256": {arm: paired.shared._sha256(Path(root) / arm / "attempts.jsonl") for arm in prep.BUILDERS}, + "provenance": contract["provenance"], "original_class_support_rule_met": False, + "original_formal_rule_changed": False, "formal_pass_gate": None, "dev_opened": False, "promotion_allowed": False} + if RESULT.exists() or (Path(root) / "summary.json").exists(): + raise RuntimeError("refusing to overwrite frozen evaluation") + paired.write_json(Path(root) / "summary.json", {"aggregate": summary, "private_transitions": transitions}) + prep.prior.natural.prior._write_json(RESULT, summary) + return summary + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--prepare-contract", action="store_true") + mode.add_argument("--run-paid", action="store_true") + mode.add_argument("--evaluate", action="store_true") + parser.add_argument("--env-file") + parser.add_argument("--max-new-calls", type=int, default=60) + args = parser.parse_args() + if args.prepare_contract: + value = prepare_contract() + print(json.dumps({"max_calls": 60, "planning_cost_cny": value["preflight"]["planning_cost_cny"]})) + elif args.run_paid: + if args.env_file: + from dotenv import load_dotenv + load_dotenv(args.env_file, override=False) + print(json.dumps({"terminal_outcomes": len(run_paid(max_new_calls=args.max_new_calls))})) + else: + value = evaluate() + print(json.dumps({k: value[k] for k in ("status", "provider_calls", "estimated_cost_cny", "primary")}, indent=2)) diff --git a/experiments/evals/reports/refusal_evidence_sufficiency/v5_3_regression_result.json b/experiments/evals/reports/refusal_evidence_sufficiency/v5_3_regression_result.json new file mode 100644 index 00000000..9cc92eeb --- /dev/null +++ b/experiments/evals/reports/refusal_evidence_sufficiency/v5_3_regression_result.json @@ -0,0 +1,348 @@ +{ + "status": "OBSERVED_DEVELOPMENT_REGRESSION_COMPLETE_NO_ACCEPTANCE", + "provider_calls": 60, + "estimated_cost_cny": 1.9172939999999998, + "primary": { + "cases": 30, + "binary_cases": 25, + "ambiguous_cases": 5, + "target_counts": { + "SUFFICIENT": 21, + "INSUFFICIENT": 4, + "AMBIGUOUS": 5 + }, + "arms": { + "v5_2_1": { + "confusion": { + "sufficient_as_sufficient": 19, + "sufficient_as_insufficient": 2, + "insufficient_as_sufficient": 1, + "insufficient_as_insufficient": 3 + }, + "valid_correct_binary": 22, + "all_case_failures": 0, + "binary_failures": 0, + "class_failures": { + "SUFFICIENT": 0, + "INSUFFICIENT": 0 + }, + "binary_agreement_including_failures": 0.88, + "binary_agreement_possible_bounds": [ + 0.88, + 0.88 + ], + "sufficient_proxy_recall": 0.9047619047619048, + "insufficient_proxy_recall": 0.75, + "admitted_all": 23, + "admission_coverage": 0.7666666666666667, + "admitted_binary": 20, + "admitted_binary_proxy_agreement": 0.95, + "ambiguous_outcomes": { + "SUFFICIENT": 3, + "INSUFFICIENT": 2, + "FAILURE": 0 + } + }, + "v5_3": { + "confusion": { + "sufficient_as_sufficient": 20, + "sufficient_as_insufficient": 1, + "insufficient_as_sufficient": 1, + "insufficient_as_insufficient": 3 + }, + "valid_correct_binary": 23, + "all_case_failures": 0, + "binary_failures": 0, + "class_failures": { + "SUFFICIENT": 0, + "INSUFFICIENT": 0 + }, + "binary_agreement_including_failures": 0.92, + "binary_agreement_possible_bounds": [ + 0.92, + 0.92 + ], + "sufficient_proxy_recall": 0.9523809523809523, + "insufficient_proxy_recall": 0.75, + "admitted_all": 24, + "admission_coverage": 0.8, + "admitted_binary": 21, + "admitted_binary_proxy_agreement": 0.9523809523809523, + "ambiguous_outcomes": { + "SUFFICIENT": 3, + "INSUFFICIENT": 2, + "FAILURE": 0 + } + } + } + }, + "common_valid_binary": { + "cases": 25, + "binary_cases": 25, + "ambiguous_cases": 0, + "target_counts": { + "SUFFICIENT": 21, + "INSUFFICIENT": 4, + "AMBIGUOUS": 0 + }, + "arms": { + "v5_2_1": { + "confusion": { + "sufficient_as_sufficient": 19, + "sufficient_as_insufficient": 2, + "insufficient_as_sufficient": 1, + "insufficient_as_insufficient": 3 + }, + "valid_correct_binary": 22, + "all_case_failures": 0, + "binary_failures": 0, + "class_failures": { + "SUFFICIENT": 0, + "INSUFFICIENT": 0 + }, + "binary_agreement_including_failures": 0.88, + "binary_agreement_possible_bounds": [ + 0.88, + 0.88 + ], + "sufficient_proxy_recall": 0.9047619047619048, + "insufficient_proxy_recall": 0.75, + "admitted_all": 20, + "admission_coverage": 0.8, + "admitted_binary": 20, + "admitted_binary_proxy_agreement": 0.95, + "ambiguous_outcomes": { + "SUFFICIENT": 0, + "INSUFFICIENT": 0, + "FAILURE": 0 + } + }, + "v5_3": { + "confusion": { + "sufficient_as_sufficient": 20, + "sufficient_as_insufficient": 1, + "insufficient_as_sufficient": 1, + "insufficient_as_insufficient": 3 + }, + "valid_correct_binary": 23, + "all_case_failures": 0, + "binary_failures": 0, + "class_failures": { + "SUFFICIENT": 0, + "INSUFFICIENT": 0 + }, + "binary_agreement_including_failures": 0.92, + "binary_agreement_possible_bounds": [ + 0.92, + 0.92 + ], + "sufficient_proxy_recall": 0.9523809523809523, + "insufficient_proxy_recall": 0.75, + "admitted_all": 21, + "admission_coverage": 0.84, + "admitted_binary": 21, + "admitted_binary_proxy_agreement": 0.9523809523809523, + "ambiguous_outcomes": { + "SUFFICIENT": 0, + "INSUFFICIENT": 0, + "FAILURE": 0 + } + } + } + }, + "all_outcome_transitions": { + "SUFFICIENT_to_SUFFICIENT": 22, + "SUFFICIENT_to_INSUFFICIENT": 1, + "SUFFICIENT_to_FAILURE": 0, + "INSUFFICIENT_to_SUFFICIENT": 2, + "INSUFFICIENT_to_INSUFFICIENT": 5, + "INSUFFICIENT_to_FAILURE": 0, + "FAILURE_to_SUFFICIENT": 0, + "FAILURE_to_INSUFFICIENT": 0, + "FAILURE_to_FAILURE": 0 + }, + "paired_binary_agreement_transitions": { + "both_agree": 21, + "baseline_only": 1, + "candidate_only": 2, + "neither_agrees": 1 + }, + "family_sensitivity": { + "drop_TRAIN_Q383": { + "cases": 29, + "binary_cases": 24, + "ambiguous_cases": 5, + "target_counts": { + "SUFFICIENT": 20, + "INSUFFICIENT": 4, + "AMBIGUOUS": 5 + }, + "arms": { + "v5_2_1": { + "confusion": { + "sufficient_as_sufficient": 18, + "sufficient_as_insufficient": 2, + "insufficient_as_sufficient": 1, + "insufficient_as_insufficient": 3 + }, + "valid_correct_binary": 21, + "all_case_failures": 0, + "binary_failures": 0, + "class_failures": { + "SUFFICIENT": 0, + "INSUFFICIENT": 0 + }, + "binary_agreement_including_failures": 0.875, + "binary_agreement_possible_bounds": [ + 0.875, + 0.875 + ], + "sufficient_proxy_recall": 0.9, + "insufficient_proxy_recall": 0.75, + "admitted_all": 22, + "admission_coverage": 0.7586206896551724, + "admitted_binary": 19, + "admitted_binary_proxy_agreement": 0.9473684210526315, + "ambiguous_outcomes": { + "SUFFICIENT": 3, + "INSUFFICIENT": 2, + "FAILURE": 0 + } + }, + "v5_3": { + "confusion": { + "sufficient_as_sufficient": 19, + "sufficient_as_insufficient": 1, + "insufficient_as_sufficient": 1, + "insufficient_as_insufficient": 3 + }, + "valid_correct_binary": 22, + "all_case_failures": 0, + "binary_failures": 0, + "class_failures": { + "SUFFICIENT": 0, + "INSUFFICIENT": 0 + }, + "binary_agreement_including_failures": 0.9166666666666666, + "binary_agreement_possible_bounds": [ + 0.9166666666666666, + 0.9166666666666666 + ], + "sufficient_proxy_recall": 0.95, + "insufficient_proxy_recall": 0.75, + "admitted_all": 23, + "admission_coverage": 0.7931034482758621, + "admitted_binary": 20, + "admitted_binary_proxy_agreement": 0.95, + "ambiguous_outcomes": { + "SUFFICIENT": 3, + "INSUFFICIENT": 2, + "FAILURE": 0 + } + } + } + }, + "drop_TRAIN_Q417": { + "cases": 29, + "binary_cases": 24, + "ambiguous_cases": 5, + "target_counts": { + "SUFFICIENT": 20, + "INSUFFICIENT": 4, + "AMBIGUOUS": 5 + }, + "arms": { + "v5_2_1": { + "confusion": { + "sufficient_as_sufficient": 18, + "sufficient_as_insufficient": 2, + "insufficient_as_sufficient": 1, + "insufficient_as_insufficient": 3 + }, + "valid_correct_binary": 21, + "all_case_failures": 0, + "binary_failures": 0, + "class_failures": { + "SUFFICIENT": 0, + "INSUFFICIENT": 0 + }, + "binary_agreement_including_failures": 0.875, + "binary_agreement_possible_bounds": [ + 0.875, + 0.875 + ], + "sufficient_proxy_recall": 0.9, + "insufficient_proxy_recall": 0.75, + "admitted_all": 22, + "admission_coverage": 0.7586206896551724, + "admitted_binary": 19, + "admitted_binary_proxy_agreement": 0.9473684210526315, + "ambiguous_outcomes": { + "SUFFICIENT": 3, + "INSUFFICIENT": 2, + "FAILURE": 0 + } + }, + "v5_3": { + "confusion": { + "sufficient_as_sufficient": 19, + "sufficient_as_insufficient": 1, + "insufficient_as_sufficient": 1, + "insufficient_as_insufficient": 3 + }, + "valid_correct_binary": 22, + "all_case_failures": 0, + "binary_failures": 0, + "class_failures": { + "SUFFICIENT": 0, + "INSUFFICIENT": 0 + }, + "binary_agreement_including_failures": 0.9166666666666666, + "binary_agreement_possible_bounds": [ + 0.9166666666666666, + 0.9166666666666666 + ], + "sufficient_proxy_recall": 0.95, + "insufficient_proxy_recall": 0.75, + "admitted_all": 23, + "admission_coverage": 0.7931034482758621, + "admitted_binary": 20, + "admitted_binary_proxy_agreement": 0.95, + "ambiguous_outcomes": { + "SUFFICIENT": 3, + "INSUFFICIENT": 2, + "FAILURE": 0 + } + } + } + } + }, + "usage": { + "v5_2_1": { + "calls": 30, + "prompt_tokens": 178030, + "completion_tokens": 21794, + "estimated_cost_cny": 0.926382, + "latency_p50_ms": 11808.792800001356 + }, + "v5_3": { + "calls": 30, + "prompt_tokens": 189550, + "completion_tokens": 23459, + "estimated_cost_cny": 0.990912, + "latency_p50_ms": 12277.463750000607 + } + }, + "contract_canonical_sha256": "db907067fc27d829d85a81cca233aaf4e8e42ac14d249963bafcdc409211cb05", + "targets_sha256": "dbea0f827c0389e695462855240c43afd0541ff72ad359b592d60817a6757953", + "journal_sha256": { + "v5_2_1": "006fb844a0f5bd1f813df2458922016aa2b3d937ccd767222693d3f7758893bf", + "v5_3": "fa9ad1a42dba0d879f3c300d66a579999f337a795766cf404ff53bf4ce71cc0b" + }, + "provenance": "AI proxy, same Qwen family as classifier; reviewer not independent or human expert gold; additional TRAIN diagnosis not holdout; all TRAIN already used in project retrieval/automated analysis; not representative enterprise traffic; conflict category absent; v5.3 designed after observing this population's predictions and targeted evidence; explicitly not unseen validation", + "original_class_support_rule_met": false, + "original_formal_rule_changed": false, + "formal_pass_gate": null, + "dev_opened": false, + "promotion_allowed": false +} diff --git a/experiments/evals/reports/refusal_evidence_sufficiency/v5_3_regression_result.md b/experiments/evals/reports/refusal_evidence_sufficiency/v5_3_regression_result.md new file mode 100644 index 00000000..08033bf6 --- /dev/null +++ b/experiments/evals/reports/refusal_evidence_sufficiency/v5_3_regression_result.md @@ -0,0 +1,101 @@ +# Observed TRAIN v5.2.1 / v5.3 development regression + +Decision: **small proxy-agreement gain, no independent acceptance or promotion**. +v5.3 corrects two proxy disagreements and introduces one, net one more agreement. +The same insufficient case is still admitted. The incident-applicability hypothesis +has no demonstrated benefit here; broader semantic reliability is not established. + +## Execution and evidence boundary + +User explicitly authorized execution under merged PR77. Follow the frozen +[protocol](v5_3_regression_protocol.md): all30 original TRAIN questions/full ordered +Top14; unchanged 21 SUFFICIENT, 4 INSUFFICIENT and 5 AMBIGUOUS AI-proxy labels. +These are Qwen drafts plus current-assistant evidence review, not independent +expert gold. Prior predictions and targeted evidence informed v5.3; this is observed +development regression, not unseen validation or representative enterprise accuracy. +The original minimum-five-per-class rule remains unmet and unchanged. + +Exactly60 calls, 30 per fresh concurrent arm; Singapore +`qwen3.5-plus-2026-04-20`, temperature0, thinking off, JSON, max2048 output tokens, +alternating arm order. Two initial calls were followed by the remaining58 from the +same checkpoint, with no repeated calls. No retries, recharge, provider/usage/schema +failures, new retrieval, annotations, relabeling, DEV access or serving changes. +Requests omit labels, review reasons, IDs, retrieval metadata and secrets. Targets +were opened for scoring only after all60 terminal outcomes and a stopped runner. + +## Counts before rates + +| Outcome relative to frozen AI proxy | v5.2.1 | v5.3 | +| --- | ---: | ---: | +| All-case failures | 0/30 | 0/30 | +| Binary target agreement, failures included | 22/25 (88%) | 23/25 (92%) | +| Sufficient targets admitted | 19/21 | 20/21 | +| Sufficient targets refused | 2/21 | 1/21 | +| Insufficient targets admitted | 1/4 | 1/4 | +| Insufficient targets refused | 3/4 | 3/4 | +| All-case admissions | 23/30 | 24/30 | +| Admitted-binary proxy agreement | 19/20 | 20/21 | +| Ambiguous admissions / refusals | 3/2 | 3/2 | + +All25 binary pairs are common-valid: both agree21, candidate only2, baseline only1, +neither1. Two sufficient-case refusals become admissions and one new sufficient-case +refusal appears; every insufficient and ambiguous decision is unchanged. Five +ambiguous cases never count as binary successes. Null-failure agreement bounds +coincide at88%/92% because there are no failures. These are proxy agreements, not +answer-accuracy measurements; no answers were generated. + +Dropping either known near-duplicate member separately gives baseline21/24 versus +candidate22/24, still net one. Both members agree in both arms. This is limited +correlation sensitivity, not a complete family partition or IID significance. +One outcome changes the insufficient-class rate by1/4. + +The contemporaneous baseline scores22/25, versus21/25 historically: one old refusal +does not recur despite the same frozen inputs/prompt/temperature0. The historical +score is not the comparison arm. Do not claim a two-case gain by comparing92% to +the old84%, or treat temperature0 as a determinism guarantee. + +## Targeted diagnosis, no post-result label repair + +One corrected decision stops refusing an exact-product search instruction because +of other search-engine documents. However, its reasoning still extrapolates from +another product using an unestablished shared-technology assumption. Agreement with +the proxy does not validate that reasoning or prove every cited source applies. +The other correction retains an explicit locale/OS-build compatibility caveat +instead of treating that documented caveat as insufficient evidence to answer. + +The new disagreement concerns a bulletin-location request with a stated desire for +details. The candidate adds a requirement to reproduce technical vulnerability +details, whereas the frozen proxy interprets the visible exact bulletin/CVE pointer +as sufficient for the location request. This is scope interpretation relative to an +AI proxy, not independent expert proof that the model or label is definitively wrong. +No label, requirement, citation or outcome is edited after seeing predictions. + +The persistent insufficient admission still connects a classloading explanation +from another application and package-isolation guidance to the actual reported +incident without establishing patch/configuration/classpath applicability. Valid +source references and an explicit applicability policy did not prevent a falsely +SUPPORTED judgment relative to the proxy. Do not claim the causal-bridge issue fixed. + +## Cost, latency and reproducibility + +Total estimated fixed-FX accounting cost **CNY1.917294**, below CNY8; not an invoice. +Use the frozen official International pricing/FX in the protocol, not an observed +discount or live exchange rate. + +| Usage | v5.2.1 | v5.3 | +| --- | ---: | ---: | +| Input/output tokens | 178,030 / 21,794 | 189,550 / 23,459 | +| Estimated cost | CNY0.926382 | CNY0.990912 | +| Median provider latency | 11.81 s | 12.28 s | + +Candidate cost is about1.070x and median latency1.040x in this run. This small +observed gain plus unchanged false admissions does not justify replacing the +experimental baseline or online policy. Do not automatically add another prompt, +retry these cases, broaden the dataset or open DEV under this exhausted authorization. + +The thin runner reuses the existing isolated durable execution engine and population +metrics. Contract/source/protocol/request/input/target fingerprints are pinned; +raw response/usage is durable before parsing. Private raw journals and per-case +targets/transitions stay ignored; public output is aggregate fingerprints/counts. +Completed replay adds zero calls; no pending, lock or staged response remains. +Historical experiment and annotation hashes remain unchanged. diff --git a/experiments/evals/reports/refusal_evidence_sufficiency/v5_3_regression_run_contract.json b/experiments/evals/reports/refusal_evidence_sufficiency/v5_3_regression_run_contract.json new file mode 100644 index 00000000..9d9d840c --- /dev/null +++ b/experiments/evals/reports/refusal_evidence_sufficiency/v5_3_regression_run_contract.json @@ -0,0 +1,393 @@ +{ + "run": "observed_train_v5_2_1_v5_3_development_regression_v1", + "status": "AUTHORIZED_NOT_RUN", + "cases": 30, + "case_order": [ + "TRAIN_Q046", + "TRAIN_Q070", + "TRAIN_Q106", + "TRAIN_Q132", + "TRAIN_Q154", + "TRAIN_Q162", + "TRAIN_Q186", + "TRAIN_Q223", + "TRAIN_Q239", + "TRAIN_Q242", + "TRAIN_Q244", + "TRAIN_Q248", + "TRAIN_Q285", + "TRAIN_Q287", + "TRAIN_Q288", + "TRAIN_Q362", + "TRAIN_Q379", + "TRAIN_Q382", + "TRAIN_Q383", + "TRAIN_Q384", + "TRAIN_Q417", + "TRAIN_Q426", + "TRAIN_Q461", + "TRAIN_Q462", + "TRAIN_Q470", + "TRAIN_Q501", + "TRAIN_Q538", + "TRAIN_Q540", + "TRAIN_Q554", + "TRAIN_Q587" + ], + "target_counts": { + "INSUFFICIENT": 4, + "SUFFICIENT": 21, + "AMBIGUOUS": 5 + }, + "binary_cases": 25, + "ambiguous_cases": 5, + "inputs_path": "data/refusal_v5_2_train/inputs.jsonl", + "targets_path": "data/refusal_v5_2_train/targets.jsonl", + "inputs_sha256": "e8082e4ee909dd8739529c6eeeedaaa3353efbe88a5207ba7ade1f4dee33d3df", + "targets_sha256": "dbea0f827c0389e695462855240c43afd0541ff72ad359b592d60817a6757953", + "annotation_freeze_canonical_sha256": "ff4625b0f635a0a968c055222f0e2401edd4e2baba42f8c74dae06bafc046a63", + "selection_canonical_sha256": "cdf6742bfc6b65625eec9eef37b04b773b3a3f2975c6fde826ae378ff762fd0d", + "original_protocol_normalized_sha256": "de1a06d394b9d082e4f27ed988071646cc419efe686c7f0b3512530a4373737d", + "source_hashes": { + "refusal_v5_2_control_runner.py": "f485b6b4533a432e44b730c1d5a20c2917d7d01e943b78419bb7d4a067c6a437", + "refusal_v5_paired_runner.py": "18c4eb1a55e0a0803b3860954e6048a587c7048420c09b8524eff9565ae0cab1", + "refusal_phase_b_runner.py": "6097cbf363c515bd569520ee622399e1251aa01b951b88cf2b6f5ec746d8fc99", + "refusal_citation_selection.py": "31c959c47608d28f4dad7be794d9a62d004a8d6ae8558255edb732828ebbf29e", + "refusal_requirement_coverage.py": "cf7972662ae081c25e56131ddb4cfd987f1f0081f480080cc1880b63d8578283", + "refusal_structured_evidence.py": "48eb7ee44af550e542a94e2a37b1daaf72669ee57a74ee8e95a27c2e725c45b8", + "refusal_development_policy.py": "43bc0b6169eeeda94d1418d77255cd71f751ef3e980fe6934b007000e7ae99d7", + "refusal_evidence_sufficiency.py": "ff14e79c09b53543d9d6f6a7d9add71eda1e3236df25af4afc2c9a73fe0de2c2", + "refusal_v5_2_controls.py": "cd016745ae212b983f90c3a5252fd6b5d61b7d744385cb099258daba9a405de9", + "refusal_v5_2_preparation.py": "08b5c8785d4b5d241fa1921280a6baaa6546a6b1f8fb88781afb766caf3bb200", + "refusal_v5_preparation.py": "2563de9aa4f9c7674a29806f6a41a3a5191336e9edf98a06d23ccac3d9fcb726", + "refusal_coverage_citation_example.py": "1b7e6841e8efd844d2c3ca1191a3618c64cae32b7008e801f827043a14d338e3", + "refusal_v5_2_1_preparation.py": "960700a4383efe2eb14560a9f5362a58e7a578f1c3c0457d5d28946019f22d60", + "refusal_v2_ai_proxy_runner.py": "0f73144341df6a13671c90b0da92f9ad5c60392f1417fda5a43e7d3dc64f18d7", + "refusal_natural_diagnostic_preparation.py": "e0e662c6e302e670735605581656aa71e2e88390ab93637aa36103577faa537c", + "refusal_question_scoped_coverage.py": "50321c44b4ff70242ddc4f4a221a45829e5105d81169dad08212ae489bd50c98", + "refusal_v5_3_preparation.py": "c8e58e626726240149f86101b6070fd08aa32a1666f7355c961e29c8d08a75f8" + }, + "preflight": { + "arms": { + "v5_2_1": { + "prompt_sha256": "ff9be51fc2f3223ce38976b39188e2bca9842c257e0014a1a2e2f9303c671e02", + "request_sha256": [ + "9843f94aed68c76b30c8139f4c4b858b7c64bce51c7677172c9f53accfcc0456", + "d7fdef5a69042ddbda95f5d01a8216cc7034d8a14048c01e5c0f2f48019e1269", + "c95a02f74ae0d4e35489ced45ea8205d3ff10b79199186ee4386b0b267add0c5", + "d377838b00244321eb153f66fbe153d7cb49e309de907238af40b87d8afe6527", + "da72ee0c61af6886315a88c9ced9241d0409ab70120841b5979060e50ad4a645", + "a17252cd9a20e0024885d6ee8e8b0ab9a51b02fabb58bec6587c8e1beabcc314", + "7b9cc54dbc3226a1eb9e608bfa2e57f4e4b0dcedf30c02225d35a21b4a6f5744", + "98fc529708dd66279f76297e9de8d901f615e31ec4efac8b4bd97d54be46d2be", + "0655b9c1172770e2018c4faf0e135b80fc2fc76540093bb798159a0d2473904f", + "df6a71233775c0dc5b9c222ce7a8af3e3e49cae3663773b96e764ea05d5ae041", + "18f41e0e7aa9e423b28249c8552e6e3221777ccf873da1354cc868f836c0cd3c", + "4627127d1dcffbfc44fdb195d71ba5356fce1b229800b2acfb50a70e6cd8e268", + "6d6626c5ea49257d6dfeda78b43ad94d65f30031d64dee05e29eb4b88599a617", + "a2400a868fe8eea9a0bb144885cc4185e591cd38a82bd5b96f6602000008763b", + "b2547a88c24027dbf25425e14339a738d11e0eac8ae488e405fbb2f6b65f588a", + "075dd03d8d451e5ba0c3300e18a6c4ec4c33472d2cec14ed2d3e46ec0fb87546", + "9d08a61c1fa79da3506adfa43deaeb4483c29ae5654e28875662a5936d1726ec", + "4db2582351c1462da35a56620019e3ba073ffefc6cd9f7e93f20e43db22578d3", + "00c424a7c20e04bf8e63da647ee518ff547b1827a86bf3a041f617365fe3d792", + "765fa0129328922efd477f8991b70fa935049981a6f89982c13ffcb20a4c7c26", + "576f4b66cc3d994ecb8ee9262836806a774052facf16b4215c6a522880edbe5b", + "ac3b98083446d207f96ef3b16f22740f7b85fc26684028f7da6a9b3417b6066b", + "06399ddb9c27e3e3c1fa2c10cf176cf8aad305c858fb1aa7cdb07c2b50108c18", + "bb7c6b106ef6efb4b86667dd7bb6c5d6a57c47d9217ea9fdf9c3a50b4b4d8d79", + "d35f62a20fadcfbc36ff0559e71570067cbdbb5014086c41f746a2de93dcd400", + "741660360c435faac4ef5c1f3f68560191f181e771688a7347f87d1afac16cd2", + "19cc8ba72b8723b9745c3e543004f244754139a3d56375804689be576c2688f1", + "9638312836fe80f6d423f95b0edf5babd6dbefe7d19595afea4b653f661b8b21", + "ee4b68a51c11591b1e91a849b20194190464c926f9b2b2894541bfa5847be486", + "a04c043e6cd6b46688dd816aa510501ed880cf43dc78fdf372bd0498370cd040" + ], + "input_byte_plus_framing_bounds": [ + 24907, + 25013, + 25714, + 23514, + 21258, + 21694, + 18485, + 23958, + 24049, + 25776, + 22015, + 25738, + 21625, + 17914, + 20998, + 20498, + 21851, + 21439, + 21149, + 21321, + 21260, + 20206, + 23619, + 23017, + 21586, + 24253, + 21512, + 24776, + 19037, + 23290 + ], + "max_output_tokens": 2048, + "planned_calls": 30, + "planning_cost_usd": 0.4160448 + }, + "v5_3": { + "prompt_sha256": "6b4f304b29aa52c45c40f0f15229795bfa3acc4da7d4ef85f8f99909002d1866", + "request_sha256": [ + "a393dd036c3351dba5280b5c071591736dda4593f261d1625880615347e6ea4d", + "3b641f79bb8343f91da12505f1f285c09da123fe003a409b8c4560ae40b6495f", + "7e279130d79e5823b23130f46def49a7e1432bb04e4cbbbad4c069ae49ab3a0b", + "622517907aff58474c74b313a2c61f39867fd1c8ebe014adc0fc2a93f9dc2eba", + "24632a56f3c8b033da81481448bce2d7a84e1e080b92e8da3402311942c8ee08", + "aefeaa6183b233fbb44f5dadf16456e2f4c157806195a2cbbd45a5ba0431a8cb", + "17df797298a3c2281aa589c519a26bcc03063ee95e9aa7671eb982437b986703", + "64c5a92f4c28ee969bfa9dcba1252ffba15f8a8ce7b81fce50b509c952f9fa97", + "a22553953bec7b4512c56a2468d33673d9eb0c6672647b490a168bb08d589b1b", + "8d9f022ff754bc24be3523b48d15655f880cf59312747b9b0bf7a4b28f3befd1", + "2f000191ee5a3b8bbba4288ddfb88c6b469e0c1a7141afc1eef08852244e9f04", + "d1a7f49f87d294fba4ae0cfec649cd032b3360de5393d24b57bcc8f2198d45c4", + "f2c782f7ba91542d3c26b19cc8ca9b7112ecd56a56983dbb2911942df8b05fc7", + "95d9c68af3e66a7696e92bc0ab093a4a7ecc512cf7f591dfe079b88d0e969fe4", + "f722324880d90cecec58bbe80951fb131c3ae8d6eb9d6d480fcaa696c1746348", + "b944c73c89d3b09557076efdf60a7e97ae6a1b863458843b4a1d6b187a4cd306", + "63dd290fadc201b54b1319523eb64294df9450573153c14aa8c42c0b898551c6", + "aa76ee99adc4a78592e8b3ea921eeaf7fd1cfecb94dea29a6b629b6fe784056a", + "f26856280b83601ad4c4010c15ecdf029770aa311bfaa5b8a0997bf3fb017437", + "c6d66e208c975aa6d4836bf9b90d68c2e83de40a8b8cf9949b7c2e98dba52fc7", + "95443f0c0cb0d14e8f2c9483d247550a5ce1bbbdc1b07c0c03742c2d90aa0826", + "724515e4c1a58a49896eb8f14ccf6c8c12f9cc86d3c29e89b4b3ae7dfdbe49cc", + "a39cc69bf2fe0f184a5e5c5a427014e8d790619ade5c94e9d9a20b8b0595aad3", + "69d845a407dce5a1b5bbe00020b098601c100fecf357d2374f4dda5f20511493", + "5e9ea3133eba2083f364f99e052417efed556ffa4c820624f76e750f8dfab6c7", + "9a53a654140e1fe523dc7a8907290a5fbeffae545636b1db6c53eff404cba37a", + "52dcb4f10504b0af63865c84810c2b4a565d7c205a89cbc905239589467527da", + "73acbd967ce79afa67d44869a1f8624f97ea40c6c4488e0cc7312da3ee66d351", + "c6ae5c5e16d16b2bbd59abbea5eedf0ff985e3cc3882af71a46d3cc40deba036", + "9e94a65f2f091c17e89ef03c1542e895a3fd454fc1cce327243bbe466ba30742" + ], + "input_byte_plus_framing_bounds": [ + 26989, + 27095, + 27796, + 25596, + 23340, + 23776, + 20567, + 26040, + 26131, + 27858, + 24097, + 27820, + 23707, + 19996, + 23080, + 22580, + 23933, + 23521, + 23231, + 23403, + 23342, + 22288, + 25701, + 25099, + 23668, + 26335, + 23594, + 26858, + 21119, + 25372 + ], + "max_output_tokens": 2048, + "planned_calls": 30, + "planning_cost_usd": 0.4410288 + } + }, + "planned_calls": 60, + "planning_cost_cny": 6.428052, + "classifier": { + "model": "qwen3.5-plus-2026-04-20", + "region": "Singapore", + "temperature": 0, + "enable_thinking": false, + "response_format": "json_object" + }, + "pricing_snapshot": { + "input_cny_per_1m_tokens": 3, + "output_cny_per_1m_tokens": 18, + "input_usd_per_1m_tokens": 0.4, + "output_usd_per_1m_tokens": 2.4, + "accounting_cny_per_usd": 7.5, + "hard_cost_cap_cny": 8, + "checked_date": "2026-10-15", + "context_band": "0