diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts index 39f9c600..b39832c3 100644 --- a/app/src/modelMeta.ts +++ b/app/src/modelMeta.ts @@ -154,6 +154,7 @@ export const MODEL_RELEASE_DATES: Record = { "gpt-6-astra": "2026-09-04", "gpt-6-sol": "2026-09-22", "gpt-6.1-sol": "2026-09-29", + "gpt-6.1-terra": "2026-10-26", "gpt-6-luna": "2026-09-22", "gpt-5.6-sol": "2026-07-09", "gpt-5.6-terra": "2026-07-09", diff --git a/policybench/config.py b/policybench/config.py index ec66b5d2..567bff84 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -36,10 +36,14 @@ # announced and available 2026-09-22) share the GPT-5.6 line's # Responses-API transport and pricing structure (cache reads at a tenth, # cache writes at 1.25x, doubled input and 1.5x output above 272k tokens). +# GPT-6.1 Sol (2026-09-29) and GPT-6.1 Terra (announced and available +# 2026-10-26) share it too, with cache reads at a twentieth of input; their +# overrides' cache_read rates carry that (eval_no_tools._gpt_responses_cache_rates). GPT_6_MODELS = { "gpt-6-astra": "gpt-6-astra", "gpt-6-sol": "gpt-6-sol", "gpt-6.1-sol": "gpt-6.1-sol", + "gpt-6.1-terra": "gpt-6.1-terra", "gpt-6-luna": "gpt-6-luna", } GPT_RESPONSES_MODELS = {**GPT_56_MODELS, **GPT_6_MODELS} @@ -174,6 +178,12 @@ # day OpenAI announced it at DevDay; the Models API lists the id with # created 2026-09-27T23:47Z). "gpt-6.1-sol": {"input": 2.0, "output": 10.0, "cache_read": 0.10}, + # gpt-6.1-terra: $1 / $5 per 1M input/output, $0.05 cached input, $1.25 + # cache writes (30-minute retention); 2x input and cache and 1.5x output + # above 272k input tokens (developers.openai.com/api/docs/models/ + # gpt-6.1-terra, read 2026-10-26, the day OpenAI announced it; the Models + # API lists the id with created 2026-10-13T16:26Z). + "gpt-6.1-terra": {"input": 1.0, "output": 5.0, "cache_read": 0.05}, "gpt-6-luna": {"input": 0.10, "output": 0.50}, "gpt-5.6-sol": {"input": 5.0, "output": 30.0}, "gpt-5.6-terra": {"input": 2.5, "output": 15.0}, diff --git a/policybench/eval_no_tools.py b/policybench/eval_no_tools.py index 402449ba..88ade66e 100644 --- a/policybench/eval_no_tools.py +++ b/policybench/eval_no_tools.py @@ -141,6 +141,22 @@ } ) + +def _gpt_responses_cache_rates(prices: dict[str, float]) -> tuple[float, float]: + """Per-token cache read and write rates of a Responses-API GPT model. + + The default is the GPT-5.6 line's structure: cache reads at a tenth of + input and cache writes at 1.25x. GPT-6.1 Sol and GPT-6.1 Terra list cache + reads at a twentieth (their ``PRICE_OVERRIDES_PER_1M`` entries), so an + override's own ``cache_read`` or ``cache_write`` rate replaces the + default. The caller doubles both above 272k prompt tokens, as it does + input. + """ + read = prices.get("cache_read", prices["input"] * 0.1) / 1_000_000 + write = prices.get("cache_write", prices["input"] * 1.25) / 1_000_000 + return read, write + + # LiteLLM's bundled model map predates the GPT-5.6 release. Without a # local entry it cannot infer that the unprefixed public API ids belong to # OpenAI, so requests fail before reaching the provider. Keep these entries @@ -152,6 +168,7 @@ _prices = PRICE_OVERRIDES_PER_1M[_model_id] _input_cost = _prices["input"] / 1_000_000 _output_cost = _prices["output"] / 1_000_000 + _cache_read_cost, _cache_write_cost = _gpt_responses_cache_rates(_prices) litellm.register_model( { _model_id: { @@ -160,10 +177,10 @@ "max_output_tokens": 128000, "input_cost_per_token": _input_cost, "output_cost_per_token": _output_cost, - "cache_read_input_token_cost": _input_cost * 0.1, - "cache_creation_input_token_cost": _input_cost * 1.25, + "cache_read_input_token_cost": _cache_read_cost, + "cache_creation_input_token_cost": _cache_write_cost, "input_cost_per_token_above_272k_tokens": _input_cost * 2, - "cache_read_input_token_cost_above_272k_tokens": (_input_cost * 0.2), + "cache_read_input_token_cost_above_272k_tokens": _cache_read_cost * 2, "output_cost_per_token_above_272k_tokens": _output_cost * 1.5, "litellm_provider": "openai", "mode": "chat", @@ -449,10 +466,10 @@ def _reconstruct_token_cost( PolicyBench's explicit prices are authoritative. LiteLLM's mutable model map is used only when no override exists, and that fallback is marked as - estimated. GPT-5.6 overrides include their documented cache and - long-context pricing. Other overrides can declare cache-read and - cache-write rates; overrides without them apply their standard rate to - the inclusive prompt-token count. + estimated. Responses-API GPT overrides take their documented cache and + long-context pricing (``_gpt_responses_cache_rates``). Other overrides + can declare cache-read and cache-write rates; overrides without them + apply their standard rate to the inclusive prompt-token count. """ if prompt_tokens is None or completion_tokens is None: return ReconstructedCost(None, None) @@ -468,14 +485,17 @@ def _reconstruct_token_cost( input_rate = rates["input"] / 1_000_000 output_rate = rates["output"] / 1_000_000 if override_name in GPT_RESPONSES_MODELS: + cache_read_rate, cache_write_rate = _gpt_responses_cache_rates(rates) if prompt_tokens > 272_000: input_rate *= 2 + cache_read_rate *= 2 + cache_write_rate *= 2 output_rate *= 1.5 uncached = int(prompt_tokens) - cached - cache_write cost = ( uncached * input_rate - + cached * input_rate * 0.1 - + cache_write * input_rate * 1.25 + + cached * cache_read_rate + + cache_write * cache_write_rate + int(completion_tokens) * output_rate ) elif "cache_read" in rates or "cache_write" in rates: diff --git a/policybench/model_cards.py b/policybench/model_cards.py index c9907331..70de2440 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -112,6 +112,22 @@ class ModelCard: "384-token non-thinking budget and stopped incomplete." ), ), + "gpt-6.1-terra": ModelCard( + litellm_id="gpt-6.1-terra", + answer_contract="tool", + thinking_budget=True, + expected_cost_per_scenario_usd=0.01, + notes=( + "Onboarded 2026-10-26, the day OpenAI announced it: forced tool " + "contract passed 3/3 (881 completion tokens) and 16/16 " + "whole-scenario (1,414 tokens) on the Responses API with " + "provider-default reasoning (effort medium, per the model page), " + "like GPT-6.1 Sol. Probed before its card existed, it got the " + "384-token non-thinking budget and stopped incomplete. A " + "two-household smoke parsed all 36 outputs for $0.02, with " + "reasoning tokens on every request." + ), + ), "gpt-6-luna": ModelCard( litellm_id="gpt-6-luna", answer_contract="tool", diff --git a/policybench/paper_results.py b/policybench/paper_results.py index 6373337a..d289c601 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -3480,6 +3480,10 @@ def last_engine_upgrade(self) -> EngineUpgrade: "gpt-6-sol": "2026-09-22", # OpenAI announced GPT-6.1 Sol at DevDay on 2026-09-29 (API id gpt-6.1-sol). "gpt-6.1-sol": "2026-09-29", + # OpenAI announced GPT-6.1 Terra on 2026-10-26, in ChatGPT, Codex and the + # API (id gpt-6.1-terra) the same day (x.com/OpenAI/status/ + # 2114780711664398349; 9to5mac.com 2026-10-26). + "gpt-6.1-terra": "2026-10-26", "gpt-6-luna": "2026-09-22", # piunikaweb.com 2026-04-17 SuperGrok beta (paid public tier) "grok-4.3": "2026-04-17", diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index 18788c03..f63d893d 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -13,6 +13,7 @@ from policybench.config import ( COUNTRY_PROGRAMS, GPT_56_MODELS, + GPT_RESPONSES_MODELS, MODELS, PRICE_OVERRIDES_PER_1M, ) @@ -25,6 +26,7 @@ _combined_cost_is_estimated, _completion_controls, _enforce_explanation_value_contract, + _gpt_responses_cache_rates, _reconstruct_token_cost, _request_explanations_once, _request_timeout_seconds, @@ -3240,6 +3242,165 @@ def cost(cached_prompt_tokens): assert cost(prompt_tokens) - 1e-12 <= cost(cached) <= cost(0) + 1e-12 +@pytest.mark.parametrize( + ( + "prompt_tokens", + "completion_tokens", + "cached_prompt_tokens", + "cache_write_prompt_tokens", + "expected", + ), + [ + # 200k prompt tokens stay below the 272k long-context line. + (200_000, 0, 0, 0, 0.2), + (0, 1_000_000, 0, 0, 5.0), + (200_000, 0, 200_000, 0, 0.01), + (200_000, 0, 0, 200_000, 0.25), + # The card review's case: the smoke's first ledger entry recorded + # 2,046 prompt tokens, 1,914 of them cached, and 1,959 completion + # tokens. At a tenth for cache reads it came to $0.0101184. + (2_046, 1_959, 1_914, 0, 0.0100227), + # Above 272k prompt tokens input and cache double; output takes 1.5x. + (300_000, 10_000, 300_000, 0, 0.105), + (300_000, 0, 0, 300_000, 0.75), + ], +) +def test_gpt_6_1_terra_override_reconstructs_token_costs( + prompt_tokens, + completion_tokens, + cached_prompt_tokens, + cache_write_prompt_tokens, + expected, +): + """GPT-6.1 Terra is priced at $1 / $5 per 1M input/output tokens, $0.05 + cached input (a twentieth, where the GPT-5.6 line reads at a tenth) and + $1.25 cache writes (developers.openai.com/api/docs/models/gpt-6.1-terra, + read 2026-10-26).""" + assert PRICE_OVERRIDES_PER_1M["gpt-6.1-terra"] == { + "input": 1.0, + "output": 5.0, + "cache_read": 0.05, + } + reconstructed = _reconstruct_token_cost( + model_name="gpt-6.1-terra", + model_id="gpt-6.1-terra", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + cache_write_prompt_tokens=cache_write_prompt_tokens, + ) + + assert reconstructed.usd == pytest.approx(expected) + assert reconstructed.is_estimated is False + + +# Each Responses-API GPT row's cache read and write rates per 1M tokens, as +# OpenAI lists them, written out here so the pricing tests do not derive their +# expectations from the helper they test (Terra card review r2): the GPT-6.1 +# rows read at a twentieth of input, the rest at the GPT-5.6 line's tenth; +# every row writes at 1.25x input. +LISTED_CACHE_RATES_PER_1M = { + "gpt-6.1-terra": (0.05, 1.25), + "gpt-6.1-sol": (0.10, 2.50), + "gpt-6-sol": (0.20, 2.50), + "gpt-6-luna": (0.01, 0.125), + "gpt-6-astra": (1.0, 12.5), + "gpt-5.6-sol": (0.5, 6.25), + "gpt-5.6-terra": (0.25, 3.125), + "gpt-5.6-luna": (0.1, 1.25), +} + + +def test_every_responses_model_has_a_listed_cache_rate(): + assert set(LISTED_CACHE_RATES_PER_1M) == set(GPT_RESPONSES_MODELS) + + +@pytest.mark.parametrize( + ("model", "read", "write"), + [(model, *rates) for model, rates in LISTED_CACHE_RATES_PER_1M.items()], +) +def test_responses_models_read_the_cache_at_their_listed_rate(model, read, write): + """Each Responses-API GPT row prices its cache at the rate its override + lists, or at the GPT-5.6 line's tenth and 1.25x when it lists none. The + two GPT-6.1 rows read at a twentieth (the bug the Terra card review found: + the reconstruction hard-coded a tenth).""" + assert _gpt_responses_cache_rates(PRICE_OVERRIDES_PER_1M[model]) == ( + pytest.approx(read / 1_000_000), + pytest.approx(write / 1_000_000), + ) + # 200k prompt tokens, all read from (or written to) the cache: below the + # 272k long-context line, so a fifth of the per-1M rate. + for cached, cache_write, rate in ((200_000, 0, read), (0, 200_000, write)): + reconstructed = _reconstruct_token_cost( + model_name=model, + model_id=GPT_RESPONSES_MODELS[model], + prompt_tokens=200_000, + completion_tokens=0, + cached_prompt_tokens=cached, + cache_write_prompt_tokens=cache_write, + ) + assert reconstructed.usd == pytest.approx(rate / 5) + + +@pytest.mark.parametrize("model", ["gpt-6.1-terra", "gpt-6.1-sol"]) +def test_litellm_registers_the_gpt_6_1_cache_read_rate(model): + """The local litellm entry carries the same cache rates as the + reconstruction, doubled above 272k prompt tokens.""" + entry = litellm.model_cost[GPT_RESPONSES_MODELS[model]] + read, write = _gpt_responses_cache_rates(PRICE_OVERRIDES_PER_1M[model]) + assert entry["cache_read_input_token_cost"] == pytest.approx(read) + assert entry["cache_read_input_token_cost_above_272k_tokens"] == pytest.approx( + 2 * read + ) + assert entry["cache_creation_input_token_cost"] == pytest.approx(write) + + +@given( + model=st.sampled_from(sorted(GPT_RESPONSES_MODELS)), + prompt_tokens=st.integers(min_value=0, max_value=1_050_000), + completion_tokens=st.integers(min_value=0, max_value=128_000), + cached_share=st.floats(min_value=0.0, max_value=1.0), + write_share=st.floats(min_value=0.0, max_value=1.0), +) +def test_responses_cost_is_the_sum_of_its_priced_token_classes( + model, prompt_tokens, completion_tokens, cached_share, write_share +): + """For every Responses-API GPT row and every usage: the cost is uncached + input, cache reads, cache writes and output, each at its listed rate (all + input classes doubled and output 1.5x above 272k prompt tokens); it is + never negative; and caching the whole prompt never costs more than + caching none of it, since every listed read rate is below input.""" + rates = PRICE_OVERRIDES_PER_1M[model] + cached = int(prompt_tokens * cached_share) + written = int((prompt_tokens - cached) * write_share) + + def cost(cached_prompt_tokens, cache_write_prompt_tokens): + return _reconstruct_token_cost( + model_name=model, + model_id=GPT_RESPONSES_MODELS[model], + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + cache_write_prompt_tokens=cache_write_prompt_tokens, + ).usd + + read, write = (r / 1_000_000 for r in LISTED_CACHE_RATES_PER_1M[model]) + long = prompt_tokens > 272_000 + scale_in, scale_out = (2, 1.5) if long else (1, 1) + expected = ( + scale_in + * ( + (prompt_tokens - cached - written) * rates["input"] / 1_000_000 + + cached * read + + written * write + ) + + scale_out * completion_tokens * rates["output"] / 1_000_000 + ) + assert cost(cached, written) == pytest.approx(expected, rel=1e-9, abs=1e-12) + assert cost(cached, written) >= 0 + assert cost(prompt_tokens, 0) <= cost(0, 0) + 1e-12 + + def test_gemini_4_argon_request_forces_the_answer_tool_and_sets_no_reasoning( mini_scenario, ): diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index b347ad1b..b7e2bfae 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -65,6 +65,7 @@ "gpt-6-astra": ("tool", None, 300, 16_384), "gpt-6-sol": ("tool", None, 300, 16_384), "gpt-6.1-sol": ("tool", None, 300, 16_384), + "gpt-6.1-terra": ("tool", None, 300, 16_384), "gpt-6-luna": ("tool", None, 300, 16_384), "openrouter/deepseek/deepseek-v4-pro-0813": ("tool", None, 2400, 98_304), "openrouter/deepseek/deepseek-v4-flash-0731": ("tool", None, 2400, 98_304), diff --git a/tests/test_paper_results.py b/tests/test_paper_results.py index 3a83dc12..b06b412c 100644 --- a/tests/test_paper_results.py +++ b/tests/test_paper_results.py @@ -101,6 +101,13 @@ def test_gemini_4_argon_release_date_is_its_public_day(): assert MODEL_RELEASE_DATES["gemini-4-argon"] == "2026-10-13" +def test_gpt_6_1_terra_release_date_is_its_announcement_day(): + """OpenAI announced GPT-6.1 Terra on 2026-10-26 and opened it in ChatGPT + (Plus, Pro, Business, Enterprise and Edu), Codex and the API (id + gpt-6.1-terra) the same day.""" + assert MODEL_RELEASE_DATES["gpt-6.1-terra"] == "2026-10-26" + + def test_app_release_dates_mirror_the_paper_registry(): """app/src/modelMeta.ts copies MODEL_RELEASE_DATES (its comment says to update both together); every runnable model has a date, and the two