From 1c1cc3b005eaf7928400a0b50a1583826ea381c8 Mon Sep 17 00:00:00 2001 From: Dmitry Khanukov Date: Tue, 10 Nov 2026 15:26:02 +0000 Subject: [PATCH] feat: add stateless minimum-effort Fable 5.2 campaign profile Signed-off-by: Dmitry Khanukov --- tests/test_openai_reasoning_items.py | 1 + tests/test_three_flow_fable52.py | 303 +++++++++++++++++++++++++++ tools/three_flow_admission.py | 12 +- tools/three_flow_campaign.py | 8 +- tools/three_flow_http.py | 31 ++- tools/three_flow_live.py | 1 + tools/three_flow_profiles.py | 17 +- tools/three_flow_selected.py | 33 ++- 8 files changed, 382 insertions(+), 24 deletions(-) create mode 100644 tests/test_three_flow_fable52.py diff --git a/tests/test_openai_reasoning_items.py b/tests/test_openai_reasoning_items.py index 9128bf1..cfaa096 100644 --- a/tests/test_openai_reasoning_items.py +++ b/tests/test_openai_reasoning_items.py @@ -55,6 +55,7 @@ def dispatch(body): transport = object.__new__(HTTPCampaignTransport) transport.provider = "openai" transport.selected_six = False + transport.november = False transport.model = MODEL transport.client = client transport.network_timeout = None diff --git a/tests/test_three_flow_fable52.py b/tests/test_three_flow_fable52.py new file mode 100644 index 0000000..e3a4448 --- /dev/null +++ b/tests/test_three_flow_fable52.py @@ -0,0 +1,303 @@ +"""Fable52 finite roster contract; all responses are synthetic SDK wires.""" + +import json + +import pytest + +from tests.test_selected_six_http import fixture +from tools.three_flow_profiles import registry, selected_roster + +PROFILE = "anthropic-fable52-same44-v1" +MODEL = "claude-fable-5-2" + + +def test_fable52_finite_roster_and_mapping(): + from tools.three_flow_campaign import CONFIGURATIONS, assignments, preflight + + assert selected_roster(PROFILE) == (("anthropic", MODEL),) + rows = registry(roster_profile=PROFILE, mode_profile="off-or-minimum-v1") + assert rows[0]["request_settings"]["thinking"] == {"type": "adaptive"} + assert rows[0]["request_settings"]["tool_choice"] == { + "type": "auto", + "disable_parallel_tool_use": True, + } + assert rows[0]["mapping"] == "three-flow-anthropic-fable52-stateless-v1" + assert rows[0]["request_settings"]["no_native_continuation"] is True + slots = assignments("fable52", PROFILE) + assert len(slots) == 44 + assert {(s["flow"], s["scenario"]) for s in slots} == set(CONFIGURATIONS) + assert preflight(PROFILE)["global_cap_usd"] is None + + +@pytest.mark.parametrize("transport_policy", [None, "legacy-v1"]) +def test_fable52_accepts_legacy_transport_policy(transport_policy): + kwargs = {} if transport_policy is None else {"transport_policy": transport_policy} + assert registry( + roster_profile=PROFILE, mode_profile="off-or-minimum-v1", **kwargs + ) == registry( + roster_profile=PROFILE, + mode_profile="off-or-minimum-v1", + transport_policy="legacy-v1", + ) + + +def test_fable52_rejects_paced_transport_policy(): + with pytest.raises(ValueError, match="November requires legacy-v1 transport policy"): + registry( + roster_profile=PROFILE, + mode_profile="off-or-minimum-v1", + transport_policy="paced-safe-errors-v1", + ) + + +def assert_projection(call, request): + from operatebench.agents import anthropic_messages as aa + + assert call["system"] == aa.request_instructions(request) + assert call["messages"] == [ + { + "role": "user", + "content": json.dumps( + dict(request.prompt), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ), + } + ] + assert call["tools"] == aa.anthropic_outcome_tools() + + +def test_fable_sdk_transcript_and_accounting(tmp_path, monkeypatch): + import httpx + + from tools.three_flow_mock import SDKMockTransport + from tools.three_flow_runtime import replay_trial, run_trial + + original = SDKMockTransport.handle + + def thinking_response(self, request): + response = original(self, request) + body = json.loads(response.content) + body["content"].insert( + 0, + { + "type": "thinking", + "thinking": "synthetic progress", + "signature": "offline-signature", + }, + ) + return httpx.Response(200, json=body) + + monkeypatch.setattr(SDKMockTransport, "handle", thinking_response) + budget, t, trial, network = fixture( + tmp_path, "anthropic", MODEL, roster_profile=PROFILE + ) + requests = [] + original_send = t.send + + def capture(request): + requests.append(request) + return original_send(request) + + monkeypatch.setattr(t, "send", capture) + try: + assert network.transports[0].calls == [] + assert t.http.timeout.connect == t.settings["connection_timeout_seconds"] == 10 + assert t.http.timeout.pool == t.settings["pool_timeout_seconds"] == 10 + assert ( + t.http.timeout.read is t.http.timeout.write is t.settings["timeout"] is None + ) + record = run_trial(trial, t, output=tmp_path / "trial") + assert record["evaluation"]["reliable"] + assert t.settings["native_continuation"] is False + assert t.settings["no_native_continuation"] is True + calls = network.transports[0].calls + assert len(calls) > 1 + for i, call in enumerate(calls): + assert call["thinking"] == {"type": "adaptive"} + assert call["output_config"] == {"effort": "low"} + assert call["tool_choice"] == { + "type": "auto", + "disable_parallel_tool_use": True, + } + assert not {"temperature", "top_p", "top_k"} & call.keys() + assert_projection(call, requests[i]) + raw = json.loads(record["provider_turns"][i]["wire"]["response_utf8"]) + assert {"thinking", "redacted_thinking"} <= { + block["type"] for block in raw["content"] + } + assert len({r.request_digest_sha256 for r in requests}) == len(requests) + for mutation in ("freeze_digest", "accumulate"): + bad = json.loads(json.dumps(calls[1])) + if mutation == "freeze_digest": + bad["system"] = calls[0]["system"] + else: + bad["messages"] = calls[0]["messages"] + bad["messages"] + with pytest.raises(AssertionError): + assert_projection(bad, requests[1]) + assert all(r["state"] == "settled" for r in budget.records.values()) + assert budget.cap is None + assert budget.totals()["remaining"] is None + assert ( + replay_trial(trial, record, expected_record_digest=record["record_digest"])[ + "provider_calls" + ] + == 0 + ) + assert ( + t.request_mapping + == registry(roster_profile=PROFILE, mode_profile="off-or-minimum-v1")[0][ + "mapping" + ] + ) + finally: + t.close() + budget.close() + + +@pytest.mark.parametrize("fault", ["multiple", "json", "usage", "model"]) +def test_fable_invalid_transport(tmp_path, fault): + from operatebench.agents.transport import ProviderFailure + from tools.three_flow_runtime import run_trial + + budget, t, trial, network = fixture( + tmp_path, "anthropic", MODEL, fault, roster_profile=PROFILE + ) + try: + with pytest.raises(ProviderFailure): + run_trial(trial, t, output=tmp_path / "trial") + assert len(network.transports[0].calls) == 1 + assert all(r["state"] == "unknown" for r in budget.records.values()) + finally: + t.close() + budget.close() + + +@pytest.mark.parametrize( + "fault", ["args", "badargs", "invalid_payload", "refusal", "no_action"] +) +def test_fable_quality_failure_no_salvage(tmp_path, fault): + from tools.three_flow_runtime import run_trial + + budget, t, trial, network = fixture( + tmp_path, "anthropic", MODEL, fault, roster_profile=PROFILE + ) + try: + record = run_trial(trial, t, output=tmp_path / "trial") + assert not record["evaluation"]["reliable"] + assert budget.totals()["unknown"] == 0 + assert network.transports[0].calls + finally: + t.close() + budget.close() + + +def test_fable52_published_profile_closed(): + from tests.test_selected_six_bounds import mistral_profile + from tools.three_flow_admission import digest, validate_profile + from tools.three_flow_profiles import ENDPOINTS + + p = mistral_profile() + p.update( + endpoint=ENDPOINTS["anthropic"], + context_max=1000000, + output_max=128000, + input_usd_per_million="16", + output_usd_per_million="40", + pricing_policy="fable52-standard-global-token-envelope-v1", + ) + p["tariff_evidence"]["category_rates"] = { + "input": "8", + "cache_read": "0.20", + "cache_write": "16", + "output": "40", + } + p["rate_evidence_sha256"] = digest(p["tariff_evidence"]) + validate_profile(p, "anthropic", model=MODEL, roster_profile=PROFILE) + validate_profile( + dict(p, execution_policy="legacy-v1"), + "anthropic", + model=MODEL, + roster_profile=PROFILE, + ) + for field, value in [ + ("output_max", 4096), + ("network_timeout_seconds", 30), + ("input_usd_per_million", "2"), + ]: + with pytest.raises(ValueError): + validate_profile( + dict(p, **{field: value}), + "anthropic", + model=MODEL, + roster_profile=PROFILE, + ) + with pytest.raises(ValueError): + validate_profile( + p, "anthropic", model=MODEL, roster_profile="selected-six-http-v1" + ) + + +@pytest.mark.parametrize("transport_policy", [None, "paced-safe-errors-v1"]) +def test_fable52_fresh_source_bound_admission_and_one_shot(tmp_path, transport_policy): + from tests.test_selected_six_bounds import mistral_profile + from tests.test_selected_six_http import Network + from tests.test_three_flow_live import cli, write + from tests.test_three_flow_live import fixture as admission_fixture + from tests.test_three_flow_optional_limits import consent + from tools import three_flow_live as live + from tools.three_flow_admission import digest, validate + from tools.three_flow_campaign import assignments + + sha = live.source_identity() + path, a, _ = admission_fixture(tmp_path, "anthropic") + slot = next( + s for s in assignments(a["campaign_id"], PROFILE) if s["flow"] == "commerce" + ) + p = mistral_profile() + p.update( + endpoint="https://api.anthropic.com/v1/messages", + context_max=1000000, + output_max=128000, + input_usd_per_million="16", + output_usd_per_million="40", + pricing_policy="fable52-standard-global-token-envelope-v1", + ) + p["tariff_evidence"]["category_rates"] = { + "input": "8", + "cache_read": "0.20", + "cache_write": "16", + "output": "40", + } + p["rate_evidence_sha256"] = digest(p["tariff_evidence"]) + a.update( + roster_profile=PROFILE, + mode_profile="off-or-minimum-v1", + registry=registry(roster_profile=PROFILE, mode_profile="off-or-minimum-v1"), + selected=[slot["trial_id"]], + pending=[slot["trial_id"]], + profiles={MODEL: p}, + cap_usd=None, + limit_policy={**consent(), "source_sha": sha}, + ) + if transport_policy is not None: + a["transport_policy"] = transport_policy + with pytest.raises( + ValueError, match="November requires legacy-v1 transport policy" + ): + validate(a, sha, synthetic=True) + write(path, a) + network = Network() + assert cli(path, a, network) == 2 + assert network.transports == [] + return + assert validate(a, sha, synthetic=True) == a + with pytest.raises(ValueError): + validate(a, "e" * 40, synthetic=True) + write(path, a) + network = Network() + assert cli(path, a, network) == 0 + count = len(network.transports) + assert cli(path, a, network) == 2 + assert len(network.transports) == count diff --git a/tools/three_flow_admission.py b/tools/three_flow_admission.py index e406526..aedc4de 100644 --- a/tools/three_flow_admission.py +++ b/tools/three_flow_admission.py @@ -27,6 +27,7 @@ from tools.three_flow_meta import validate_meta_profile from tools.three_flow_profiles import ( ENDPOINTS, + FABLE52_PROFILE, GEMINI_ROSTER_PROFILE, GEMINI_UNCAPPED_PROFILE, META_ROSTER_PROFILE, @@ -115,7 +116,7 @@ def proposal(sha: str, roster_profile: str = "legacy-v1") -> dict[str, Any]: ], } - if roster_profile in (SELECTED_SIX_PROFILE, NOVEMBER_PROFILE): + if roster_profile in (SELECTED_SIX_PROFILE, NOVEMBER_PROFILE, FABLE52_PROFILE): result["cap_usd"] = None if roster_profile == GEMINI_ROSTER_PROFILE: result["cap_usd"] = None @@ -193,6 +194,7 @@ def _validate( META_ROSTER_PROFILE, SELECTED_SIX_PROFILE, NOVEMBER_PROFILE, + FABLE52_PROFILE, ) or admission.get("mode_profile") != "off-or-minimum-v1" ): @@ -490,7 +492,11 @@ def validate_historical_identity(a: dict[str, Any]) -> None: # selected_registry has always omitted output_limit. Select the recorded # schema by the original roster; keep every other historical schema closed. string_fields = {"endpoint", "mapping", "settings_source"} - if a.get("roster_profile") not in (SELECTED_SIX_PROFILE, NOVEMBER_PROFILE): + if a.get("roster_profile") not in ( + SELECTED_SIX_PROFILE, + NOVEMBER_PROFILE, + FABLE52_PROFILE, + ): string_fields.add("output_limit") fields = string_fields | { "provider", @@ -758,7 +764,7 @@ def validate_profile( model: str | None = None, roster_profile: str = "legacy-v1", ) -> None: - if roster_profile in (SELECTED_SIX_PROFILE, NOVEMBER_PROFILE): + if roster_profile in (SELECTED_SIX_PROFILE, NOVEMBER_PROFILE, FABLE52_PROFILE): from tools.three_flow_selected import validate_published_profile validate_published_profile(p, provider, model, roster_profile=roster_profile) diff --git a/tools/three_flow_campaign.py b/tools/three_flow_campaign.py index 1bb2eaf..0b92f77 100644 --- a/tools/three_flow_campaign.py +++ b/tools/three_flow_campaign.py @@ -37,6 +37,7 @@ EXCLUDED_FUTURE_MODEL as EXCLUDED_FUTURE_MODEL, ) from tools.three_flow_profiles import ( + FABLE52_PROFILE, GEMINI_ROSTER_PROFILE, META_ROSTER_PROFILE, NOVEMBER_PROFILE, @@ -85,7 +86,12 @@ def preflight(roster_profile: str = "legacy-v1") -> dict[str, Any]: "source_commit": source_commit(), "global_cap_usd": None if roster_profile - in (GEMINI_ROSTER_PROFILE, SELECTED_SIX_PROFILE, NOVEMBER_PROFILE) + in ( + GEMINI_ROSTER_PROFILE, + SELECTED_SIX_PROFILE, + NOVEMBER_PROFILE, + FABLE52_PROFILE, + ) else "990" if roster_profile == META_ROSTER_PROFILE else "1000", diff --git a/tools/three_flow_http.py b/tools/three_flow_http.py index b9ce1ad..9c05e06 100644 --- a/tools/three_flow_http.py +++ b/tools/three_flow_http.py @@ -54,9 +54,7 @@ ENDPOINTS as ENDPOINTS, ) from tools.three_flow_profiles import ( - LATEST_HTTP_MODELS as LATEST_HTTP_MODELS, -) -from tools.three_flow_profiles import ( + FABLE52_MODELS, MODE_PROFILES, NOVEMBER_MODELS, SELECTED_SIX, @@ -65,6 +63,9 @@ http_reasoning_mode, selection_for, ) +from tools.three_flow_profiles import ( + LATEST_HTTP_MODELS as LATEST_HTTP_MODELS, +) from tools.three_flow_profiles import ( MODELS as MODELS, ) @@ -387,8 +388,12 @@ def __init__( self.request_mapping = ( "three-flow-anthropic-selected-six-single-decision-v2" ) - if self.november: - self.request_mapping = "three-flow-anthropic-november-stateless-v1" + if self.anthropic_stateless: + self.request_mapping = ( + "three-flow-anthropic-fable52-stateless-v1" + if (provider, model) in FABLE52_MODELS + else "three-flow-anthropic-november-stateless-v1" + ) self.guard, self.max_output_tokens = guard, max_output_tokens self.wire = BudgetWire(inner, guard, ENDPOINTS[provider], operation=selection.api) self.wire.sensitive_response = api_key.encode() @@ -416,7 +421,7 @@ def __init__( ) ) self.client: Any - if self.november: + if self.anthropic_stateless: self.http.timeout = (10, None, None, 10) if provider in ("openai", "meta", "xai", "river_http"): self.client = explicit_sdk_client( @@ -570,7 +575,7 @@ def __init__( self.settings["reasoning_mode"] = ( "MINIMUM" if model == "GLM-5.3" else "OFF" ) - if self.november: + if self.anthropic_stateless: self.settings.update( tool_choice={"type": "auto", "disable_parallel_tool_use": True}, native_continuation=False, @@ -581,6 +586,10 @@ def __init__( self.last_turn: dict[str, Any] | None = None self.flight = SingleFlight(adapter="three-flow HTTP") + @property + def anthropic_stateless(self) -> bool: + return self.november or (self.provider, self.model) in FABLE52_MODELS + def prepare_request(self, request: ModelRequest) -> ModelRequest: if not getattr(self.guard, "diagnostic", False): return request @@ -643,7 +652,7 @@ def payload(self, request: ModelRequest) -> dict[str, Any]: {"type": "any", "disable_parallel_tool_use": True} if self.selected_six else {"type": "auto", "disable_parallel_tool_use": True} - if self.november + if self.anthropic_stateless else {"type": "auto"} ), "thinking": {"type": "adaptive"}, @@ -713,18 +722,18 @@ def _dispatch(self, payload: dict[str, Any]) -> tuple[Any, ModelResponse]: raw = self.client.messages.with_raw_response.create( **ap.anthropic_sdk_kwargs(payload), timeout=self.http.timeout - if self.selected_six or self.november + if self.selected_six or self.anthropic_stateless else self.network_timeout, ) wire = WireResponse(raw.text(), raw.parse, kind="Fable campaign") # Validate hidden reasoning blocks, then use the unchanged legacy # validator on the semantic text/tool projection. Full raw retained. - if self.selected_six or self.november: + if self.selected_six or self.anthropic_stateless: from tools.three_flow_selected import check_anthropic_reading check_anthropic_reading(wire) body = dict(wire.wire) - if self.selected_six or self.november: + if self.selected_six or self.anthropic_stateless: body["usage"] = dict(body["usage"]) body["usage"].pop("speed", None) content = body.get("content") diff --git a/tools/three_flow_live.py b/tools/three_flow_live.py index 322dcd5..6fa4ea3 100644 --- a/tools/three_flow_live.py +++ b/tools/three_flow_live.py @@ -404,6 +404,7 @@ def main(argv: list[str] | None = None, *, network: Any = None) -> int: "latest-http-v1", "selected-six-http-v1", "november-model-expansion-v1", + "anthropic-fable52-same44-v1", "meta-standard-same44-v1", "gemini-standard-same44-low-v1", ), diff --git a/tools/three_flow_profiles.py b/tools/three_flow_profiles.py index c0565dc..f38749c 100644 --- a/tools/three_flow_profiles.py +++ b/tools/three_flow_profiles.py @@ -68,6 +68,8 @@ def minimum_effort(model: str) -> str: NOVEMBER_PROFILE = "november-model-expansion-v1" NOVEMBER_MODELS = (("anthropic", "claude-sonnet-5-5"),) +FABLE52_PROFILE = "anthropic-fable52-same44-v1" +FABLE52_MODELS = (("anthropic", "claude-fable-5-2"),) SELECTED_SIX_PROFILE = "selected-six-http-v1" SELECTED_SIX = ( @@ -85,6 +87,8 @@ def minimum_effort(model: str) -> str: def selected_roster(roster_profile: str = "legacy-v1") -> tuple[tuple[str, str], ...]: + if roster_profile == FABLE52_PROFILE: + return FABLE52_MODELS if roster_profile == NOVEMBER_PROFILE: return NOVEMBER_MODELS if roster_profile == SELECTED_SIX_PROFILE: @@ -140,7 +144,7 @@ def selection_for( ) -> CampaignSelection | None: """Resolve only existing valid cells; callers retain their validation order.""" if mode_profile == "off-or-minimum-v1" and ( - (provider, model) in SELECTED_SIX + NOVEMBER_MODELS + (provider, model) in SELECTED_SIX + NOVEMBER_MODELS + FABLE52_MODELS ): return CampaignSelection( provider, @@ -249,7 +253,8 @@ def registry( transport_policy: str = "legacy-v1", roster_profile: str = "legacy-v1", ) -> list[dict[str, Any]]: - if roster_profile == NOVEMBER_PROFILE: + if roster_profile in (NOVEMBER_PROFILE, FABLE52_PROFILE): + fable = roster_profile == FABLE52_PROFILE if mode_profile != "off-or-minimum-v1": raise ValueError("November requires qualified minimum mode") if transport_policy != "legacy-v1": @@ -259,7 +264,11 @@ def registry( "provider": p, "model": m, "endpoint": ENDPOINTS[p], - "mapping": "three-flow-anthropic-november-stateless-v1", + "mapping": ( + "three-flow-anthropic-fable52-stateless-v1" + if fable + else "three-flow-anthropic-november-stateless-v1" + ), "request_settings": { **http_mode_fields(p, m, mode_profile), "tool_choice": {"type": "auto", "disable_parallel_tool_use": True}, @@ -269,7 +278,7 @@ def registry( "retries": 0, "admitted": False, } - for p, m in NOVEMBER_MODELS + for p, m in (FABLE52_MODELS if fable else NOVEMBER_MODELS) ] if roster_profile == SELECTED_SIX_PROFILE: if mode_profile != "off-or-minimum-v1": diff --git a/tools/three_flow_selected.py b/tools/three_flow_selected.py index 27871f6..0ad4c61 100644 --- a/tools/three_flow_selected.py +++ b/tools/three_flow_selected.py @@ -12,6 +12,8 @@ from operatebench.providers.wire import WireResponse, wire_invalid from tools.three_flow_profiles import ( ENDPOINTS, + FABLE52_MODELS, + FABLE52_PROFILE, NOVEMBER_MODELS, NOVEMBER_PROFILE, SELECTED_SIX, @@ -343,8 +345,9 @@ def validate_published_profile( from tools.three_flow_admission import GUARANTEES, digest + fable = roster_profile == FABLE52_PROFILE november = roster_profile == NOVEMBER_PROFILE - roster = NOVEMBER_MODELS if november else SELECTED_SIX + roster = FABLE52_MODELS if fable else NOVEMBER_MODELS if november else SELECTED_SIX fields = { "endpoint", "input_max", @@ -368,18 +371,32 @@ def validate_published_profile( raise ValueError("closed selected-six published profile required") if ( p["pricing_policy"] - != ("november-published-token-envelope-v1" if november else PRICING_POLICY) + != ( + "fable52-standard-global-token-envelope-v1" + if fable + else "november-published-token-envelope-v1" + if november + else PRICING_POLICY + ) or p["accounting_policy"] != ACCOUNTING_POLICY or p["guarantees"] != dict.fromkeys(GUARANTEES, False) ): raise ValueError("selected-six estimate cannot assert billing guarantees") - if november: + if fable: + if p.get("execution_policy") not in ( + "legacy-v1", + "three-flow-optional-limits-v1", + ): + raise ValueError("Fable52 requires an existing monetary execution policy") + elif november: if p.get("execution_policy") != "three-flow-optional-limits-v1": raise ValueError("November requires null monetary policy") else: validate_selected_profile(p, provider, model) assert model is not None - context, input_max, output = (1000000, None, 128000) if november else BOUNDS[model] + context, input_max, output = ( + (1000000, None, 128000) if november or fable else BOUNDS[model] + ) if ( (p["context_max"], p["input_max"], p["output_max"]) != (context, input_max, output) @@ -387,7 +404,13 @@ def validate_published_profile( or p["endpoint"] != ENDPOINTS[provider] ): raise ValueError("selected-six published route/bounds differ") - rates = ("2", "0.10", "4", "10") if november else TARIFFS[model] + rates = ( + ("8", "0.20", "16", "40") + if fable + else ("2", "0.10", "4", "10") + if november + else TARIFFS[model] + ) if ( p["input_usd_per_million"] != str(max(Decimal(v) for v in rates[:3] if v is not None))