From 53a3e0f4d41f6c0da3147901f03f0d0e2b1903d0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 13 Nov 2026 13:24:55 -0500 Subject: [PATCH 1/7] Add Gemini 4 Carbon's and Gemini 4 Hydrogen's cards, list prices and release dates Adds the serving card, list price and release date for Google's Gemini 4 Carbon and Gemini 4 Hydrogen so the board can run them. No result: the models join the board in a later release, and this changes no published artifact. - Release: the Gemini API changelog lists gemini-4-carbon as generally available on 2026-11-10 and gemini-4-hydrogen on 2026-11-12 (read 2026-11-13). Carbon's earlier release to cybersecurity defenders through the Fairwind Program is excluded under the public-availability rule. Route: the first-party Gemini API, gemini/gemini-4-carbon and gemini/gemini-4-hydrogen. - Price: Carbon $2 / $10 per 1M input/output tokens and $0.10 cached input, the rate Google lists through 2026-11-30 (it doubles on 2026-12-01; the override holds the rate through 2026-11-30). Hydrogen $1.50 / $7.50 and $0.075 cached input, with no dated change. - Cards: written from Google's documentation, read 2026-11-13; they record no request to either model. Both take the board's forced answer tool on Gemini 4 Argon's pattern, send no reasoning parameter (Google gives a default thinking level of medium for Carbon and high for Hydrogen) and take the thinking class's 300s timeout. Neither card sets a per-scenario cost. - Ceiling: 32,768 completion tokens for Carbon and 65,536 for Hydrogen, what one non-streaming generateContent request returns per Google's long-decode and thinking guides and the Models API. - The frontier-column list is unchanged. Co-Authored-By: Claude Fable 5.2 --- app/src/modelMeta.ts | 2 + policybench/config.py | 38 +++++++++ policybench/model_cards.py | 101 ++++++++++++++++++++++ policybench/paper_results.py | 16 ++++ tests/test_eval_no_tools.py | 161 +++++++++++++++++++++++++++++++++++ tests/test_model_cards.py | 4 + tests/test_paper_results.py | 17 ++++ 7 files changed, 339 insertions(+) diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts index 3f4d03a1..a605bcb6 100644 --- a/app/src/modelMeta.ts +++ b/app/src/modelMeta.ts @@ -152,6 +152,8 @@ export const MODEL_RELEASE_DATES: Record = { "gemini-3.7-flash": "2026-08-13", "gemini-3.5-flash-lite": "2026-07-21", "gemini-4-argon": "2026-10-13", + "gemini-4-carbon": "2026-11-10", + "gemini-4-hydrogen": "2026-11-12", "gemini-3.1-flash-lite-preview": "2026-03-03", "gpt-5.4-mini": "2026-03-17", "gpt-5.4-nano": "2026-03-17", diff --git a/policybench/config.py b/policybench/config.py index 693c25b1..43b30dda 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -103,6 +103,22 @@ # the first-party Gemini API like the other Gemini rows. The id is the # model page's one stable version; Google lists no dated snapshot. "gemini-4-argon": "gemini/gemini-4-argon", + # Gemini 4 Carbon. Google opened it to the public on 2026-11-10, when the + # Gemini API changelog lists gemini-4-carbon as generally available + # (aistudio.google.com/docs/changelog, read 2026-11-13); Google's post of + # that day says it had released the model two weeks earlier to + # cybersecurity defenders through its Fairwind Program. It runs on the + # first-party Gemini API like the other Gemini rows. The id is the model + # page's one stable version; Google lists no dated snapshot + # (aistudio.google.com/docs/models/gemini-4-carbon, read 2026-11-13). + "gemini-4-carbon": "gemini/gemini-4-carbon", + # Gemini 4 Hydrogen. The Gemini API changelog lists gemini-4-hydrogen as + # generally available on 2026-11-12, on the free and the paid tier + # (aistudio.google.com/docs/changelog, read 2026-11-13). It runs on the + # first-party Gemini API like the other Gemini rows. The id is the model + # page's one stable version; Google lists no dated snapshot + # (aistudio.google.com/docs/models/gemini-4-hydrogen, read 2026-11-13). + "gemini-4-hydrogen": "gemini/gemini-4-hydrogen", # Cloaked OpenRouter preview listed 2026-08-20 (free window); Z.ai later # identified it as GLM-5.3-Flash. On the board as a labeled preview row. "ox-alpha": "openrouter/stealth/ox-alpha", @@ -245,6 +261,28 @@ # three rates (read 2026-10-13); the override keeps the row priced if the # map is unavailable or changes on 2026-12-01. "gemini-4-argon": {"input": 2.0, "output": 10.0, "cache_read": 0.10}, + # gemini-4-carbon: $2 / $10 per 1M input/output tokens (output includes + # thinking tokens) and $0.10 cached input, the standard-tier rates Google + # lists "through November 30, 2026"; from 2026-12-01 they double to + # $4 / $20 and $0.20 (aistudio.google.com/docs/pricing, read 2026-11-13). + # The row is priced at the rate in force on its run day, as + # gemini-4-argon is at the same dated rates; this entry holds the rate + # through 2026-11-30. The page lists one rate at every prompt length, + # implicit caching only, and no free tier for this model. litellm + # 1.95.0's bundled price map has no gemini-4 id, and its remote map + # carries the same three introductory rates (read 2026-11-13); the + # override keeps the row priced if the map is unavailable or changes on + # 2026-12-01. + "gemini-4-carbon": {"input": 2.0, "output": 10.0, "cache_read": 0.10}, + # gemini-4-hydrogen: $1.50 / $7.50 per 1M input/output tokens (output + # includes thinking tokens) and $0.075 cached input, the paid tier's + # standard rates, with no dated change (aistudio.google.com/docs/pricing, + # read 2026-11-13). The page lists one rate at every prompt length and + # implicit caching only; it also lists a free tier, which the row's cost + # does not use. litellm 1.95.0's bundled price map has no gemini-4 id, and + # its remote map carries the same three rates (read 2026-11-13); the + # override keeps the row priced if the map is unavailable. + "gemini-4-hydrogen": {"input": 1.5, "output": 7.5, "cache_read": 0.075}, # glm-5.3: $1.40 / $4.40 per 1M (OpenRouter list, 2026-09-03). "glm-5.3": {"input": 1.40, "output": 4.40}, # deepseek dated snapshots via OpenRouter (list prices 2026-09-03). diff --git a/policybench/model_cards.py b/policybench/model_cards.py index 9a10e6be..3e7531ea 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -570,6 +570,107 @@ class ModelCard: "shows only that the parameter is accepted." ), ), + # Gemini 4 Carbon. This card is written from Google's documentation, read + # 2026-11-13, and from Gemini 4 Argon's card above. It records no request + # to the model. Field by field: + # - answer_contract "tool": Argon's contract, the board's forced answer + # tool. Google's migration guide says that upgrading from + # gemini-4-argon to gemini-4-carbon "requires no request schema + # changes" (aistudio.google.com/docs/whats-new-gemini-4-carbon), and + # litellm sends the forced call as it does for Argon (mode ANY with + # allowed_function_names). + # - explanation_chunk_size None: the canonical whole-scenario request + # (chunking is closed to new models). + # - thinking_budget True: the model reasons by default. Google's thinking + # guide lists its default as "On (medium)", with levels low, medium and + # high (aistudio.google.com/docs/generate-content/thinking), so it is + # in the thinking-default class and takes that class's 300s timeout. + # The 16,384-token starting budget is the Gemini family's. + # - request_timeout_seconds None: the class default, as on Argon. + # - provider_max_completion_tokens 32_768: what one non-streaming request + # returns. Google's long-decode guide gives 32K per non-streaming + # request on gemini-4-carbon and gemini-4-argon + # (aistudio.google.com/docs/long-decode), its thinking guide gives + # 32,768 tokens for the same calls, and the Models API's + # outputTokenLimit for the model is 32,768 (models.get, read + # 2026-11-13). The model page lists 65,536 as "default synchronous", + # which the thinking guide gives for streaming calls on this model. + # litellm makes one non-streaming REST request. + # - expected_cost_per_scenario_usd is not set: no per-scenario cost has + # been measured. + "gemini/gemini-4-carbon": ModelCard( + litellm_id="gemini/gemini-4-carbon", + answer_contract="tool", + thinking_budget=True, + provider_max_completion_tokens=32_768, + notes=( + "Written 2026-11-13 from Google's documentation, read that day; " + "the card records no request to the model. Google opened the " + "model to the public on 2026-11-10 (the Gemini API changelog " + "lists gemini-4-carbon as generally available that day; an " + "earlier release went to cybersecurity defenders through its " + "Fairwind Program). The card takes Gemini 4 Argon's treatment, " + "the board's forced answer tool on the whole-scenario request: " + "Google's migration guide says that upgrading from " + "gemini-4-argon to gemini-4-carbon needs no change to the " + "request schema. The harness sends no thinkingConfig, and Google's " + "thinking guide lists the model's default as On (medium), with " + "levels low, medium and high. That guide and the long-decode " + "guide give 32,768 tokens for one non-streaming request on this " + "model, which is what the harness sends, and the Models API's " + "outputTokenLimit is 32,768; the model page lists 65,536 as the " + "default synchronous output. Google's guide says Carbon spends " + "more thinking tokens per turn than Argon. Google lists $2/$10 " + "per 1M through 2026-11-30 and $4/$20 from 2026-12-01." + ), + ), + # Gemini 4 Hydrogen. Like the Carbon card above, this card is written + # from Google's documentation, read 2026-11-13, and records no request to + # the model. Field by field: + # - answer_contract "tool": the board's forced answer tool, the contract + # of every Gemini card from Gemini 3.6 Flash on. Google's model page + # lists function calling as supported + # (aistudio.google.com/docs/models/gemini-4-hydrogen). + # - explanation_chunk_size None: the canonical whole-scenario request + # (chunking is closed to new models). + # - thinking_budget True: the model reasons by default. Google's thinking + # guide lists its default as "On (high)", with levels low, medium, high + # and xhigh (aistudio.google.com/docs/generate-content/thinking), so it + # is in the thinking-default class and takes that class's 300s timeout. + # The 16,384-token starting budget is the Gemini family's. The harness + # sends no thinkingConfig, so this model's default is high where + # Carbon's and Argon's is medium. + # - request_timeout_seconds None: the class default. + # - provider_max_completion_tokens 65_536: what one non-streaming request + # returns. Google's long-decode and thinking guides give 65,536 tokens + # for non-streaming gemini-4-hydrogen calls + # (aistudio.google.com/docs/long-decode), and the Models API's + # outputTokenLimit for the model is 65,536 (models.get, read + # 2026-11-13). + # - expected_cost_per_scenario_usd is not set: no per-scenario cost has + # been measured. + "gemini/gemini-4-hydrogen": ModelCard( + litellm_id="gemini/gemini-4-hydrogen", + answer_contract="tool", + thinking_budget=True, + provider_max_completion_tokens=65_536, + notes=( + "Written 2026-11-13 from Google's documentation, read that day; " + "the card records no request to the model. The Gemini API " + "changelog lists gemini-4-hydrogen as generally available on " + "2026-11-12, on the free and the paid tier. The card takes the " + "board's forced answer tool on the whole-scenario request, as " + "every Gemini card from Gemini 3.6 Flash on does; Google's " + "model page lists function calling as supported. The harness " + "sends no thinkingConfig, and Google's thinking guide lists the " + "model's default as On (high), with levels low, medium, high " + "and xhigh, where Gemini 4 Carbon's and Argon's default is " + "medium. That guide and the long-decode guide give 65,536 " + "tokens for one non-streaming request on this model, and the " + "Models API's outputTokenLimit is 65,536. Google lists $1.50/" + "$7.50 per 1M." + ), + ), "openrouter/z-ai/glm-5.3": ModelCard( litellm_id="openrouter/z-ai/glm-5.3", answer_contract="tool", diff --git a/policybench/paper_results.py b/policybench/paper_results.py index b0fa50c4..9297b3fc 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -3621,6 +3621,22 @@ def last_engine_upgrade(self) -> EngineUpgrade: # Program cyber partners only (techcrunch.com 2026-09-30); that day is # excluded under the public-availability rule. "gemini-4-argon": "2026-10-13", + # aistudio.google.com/docs/changelog ("November 10, 2026": Gemini 4 + # Carbon, gemini-4-carbon, generally available) and blog.google/ + # innovation-and-ai/models-and-research/gemini-models/ + # gemini-4-carbon-now-available (2026-11-10: the API in Google AI Studio + # "for all paying developers and enterprises", and Google AI Ultra + # subscribers in the Gemini app), both read 2026-11-13. The post says + # Google had released the model two weeks earlier to cybersecurity + # defenders through its Fairwind Program; that partner-only period is + # excluded under the public-availability rule. + "gemini-4-carbon": "2026-11-10", + # aistudio.google.com/docs/changelog ("November 12, 2026": Gemini 4 + # Hydrogen, gemini-4-hydrogen, generally available, on the free tier and + # the paid tier) and blog.google/innovation-and-ai/models-and-research/ + # gemini-models/introducing-gemini-4-hydrogen (2026-11-12: "Starting + # today" in the Gemini API and the Gemini app), both read 2026-11-13. + "gemini-4-hydrogen": "2026-11-12", # blog.google gemini-3-1-flash-lite; siliconangle.com 2026-03-03 "gemini-3.1-flash-lite-preview": "2026-03-03", # openai.com/index/introducing-gpt-5-4-mini-and-nano; 9to5mac 2026-03-17 diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index ce00440d..c9247eb1 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -3281,6 +3281,128 @@ def cost(cached_prompt_tokens): assert cost(prompt_tokens) - 1e-12 <= cost(cached) <= cost(0) + 1e-12 +def test_gemini_4_carbon_runs_first_party_at_googles_dated_rate(): + """Gemini 4 Carbon runs on the first-party Gemini API, like the other + Gemini rows, at the rate Google's pricing page lists through 2026-11-30: + $2 / $10 per 1M input/output tokens and $0.10 cached input, at every + prompt length (the page doubles all three from 2026-12-01). The override + holds that one rate and the reconstruction takes no date, so the entry + prices a request at these rates whenever it is made.""" + assert MODELS["gemini-4-carbon"] == "gemini/gemini-4-carbon" + assert PRICE_OVERRIDES_PER_1M["gemini-4-carbon"] == { + "input": 2.0, + "output": 10.0, + "cache_read": 0.10, + } + + +@pytest.mark.parametrize( + ("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"), + [ + (1_000_000, 0, 0, 2.0), + (0, 1_000_000, 0, 10.0), + (1_000_000, 1_000_000, 0, 12.0), + (1_000_000, 0, 1_000_000, 0.10), + (1_000_000, 0, 250_000, 1.525), + ], +) +def test_gemini_4_carbon_override_reconstructs_token_costs( + prompt_tokens, completion_tokens, cached_prompt_tokens, expected +): + reconstructed = _reconstruct_token_cost( + model_name="gemini-4-carbon", + model_id="gemini/gemini-4-carbon", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ) + + assert reconstructed.usd == pytest.approx(expected) + assert reconstructed.is_estimated is False + + +def test_gemini_4_hydrogen_runs_first_party_at_googles_list_rate(): + """Gemini 4 Hydrogen runs on the first-party Gemini API, like the other + Gemini rows, at the paid-tier rate on Google's pricing page: $1.50 / + $7.50 per 1M input/output tokens and $0.075 cached input, at every prompt + length, with no dated change.""" + assert MODELS["gemini-4-hydrogen"] == "gemini/gemini-4-hydrogen" + assert PRICE_OVERRIDES_PER_1M["gemini-4-hydrogen"] == { + "input": 1.5, + "output": 7.5, + "cache_read": 0.075, + } + + +@pytest.mark.parametrize( + ("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"), + [ + (1_000_000, 0, 0, 1.5), + (0, 1_000_000, 0, 7.5), + (1_000_000, 1_000_000, 0, 9.0), + (1_000_000, 0, 1_000_000, 0.075), + (1_000_000, 0, 250_000, 1.14375), + ], +) +def test_gemini_4_hydrogen_override_reconstructs_token_costs( + prompt_tokens, completion_tokens, cached_prompt_tokens, expected +): + reconstructed = _reconstruct_token_cost( + model_name="gemini-4-hydrogen", + model_id="gemini/gemini-4-hydrogen", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ) + + assert reconstructed.usd == pytest.approx(expected) + assert reconstructed.is_estimated is False + + +# Google's listed per-1M rates (input, cached input, output), written out +# here and not read from PRICE_OVERRIDES_PER_1M, so the property below checks +# the override against the page's figures. +GEMINI_4_LISTED_RATES_PER_1M = { + "gemini-4-carbon": (2.0, 0.10, 10.0), + "gemini-4-hydrogen": (1.5, 0.075, 7.5), +} + + +@given( + model=st.sampled_from(sorted(GEMINI_4_LISTED_RATES_PER_1M)), + prompt_tokens=st.integers(min_value=0, max_value=1_048_576), + completion_tokens=st.integers(min_value=0, max_value=131_072), + cached_share=st.floats(min_value=0.0, max_value=1.0), +) +def test_gemini_4_carbon_and_hydrogen_costs_are_linear_and_bounded_by_the_cache_rate( + model, prompt_tokens, completion_tokens, cached_share +): + """Invariants of each override's reconstruction: the cost is the sum of + uncached input, cached input and output at Google's listed rates; it is + never negative; and it lies between the all-cached and the uncached cost + of the same call.""" + input_rate, cached_rate, output_rate = GEMINI_4_LISTED_RATES_PER_1M[model] + cached = int(prompt_tokens * cached_share) + + def cost(cached_prompt_tokens): + return _reconstruct_token_cost( + model_name=model, + model_id=MODELS[model], + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ).usd + + expected = ( + (prompt_tokens - cached) * input_rate + + cached * cached_rate + + completion_tokens * output_rate + ) / 1_000_000 + assert cost(cached) == pytest.approx(expected, abs=1e-12) + assert cost(cached) >= 0 + assert cost(prompt_tokens) - 1e-12 <= cost(cached) <= cost(0) + 1e-12 + + @pytest.mark.parametrize( ( "prompt_tokens", @@ -3476,6 +3598,45 @@ def test_gemini_4_argon_request_forces_the_answer_tool_and_sets_no_reasoning( assert reasoning_key not in kwargs +@pytest.mark.parametrize( + "model_id", ["gemini/gemini-4-carbon", "gemini/gemini-4-hydrogen"] +) +def test_gemini_4_carbon_and_hydrogen_force_the_answer_tool_and_set_no_reasoning( + mini_scenario, model_id +): + """Gemini 4 Carbon's and Gemini 4 Hydrogen's cards select the board's + forced answer tool, as Gemini 4 Argon's does (the older Gemini rows + answer as JSON by the family default). The request carries no reasoning + parameter, so each model reasons at Google's default, which Google's + thinking guide gives as medium for Carbon and high for Hydrogen. Each + takes the thinking-default class's 300s timeout and the Gemini family's + 16,384-token starting budget.""" + from policybench.eval_no_tools import ( + ANSWER_FUNCTION_NAME, + _answer_contract_for_model, + _chat_completion_request_kwargs, + ) + + assert _answer_contract_for_model(model_id, env={}) == "tool" + assert _thinking_configuration(model_id) == {"mode": "provider_default"} + assert _request_timeout_seconds(model_id) == 300 + + _, kwargs = _chat_completion_request_kwargs(mini_scenario, ["income_tax"], model_id) + + assert kwargs["tool_choice"] == { + "type": "function", + "function": {"name": ANSWER_FUNCTION_NAME}, + } + assert [tool["function"]["name"] for tool in kwargs["tools"]] == [ + ANSWER_FUNCTION_NAME + ] + assert kwargs["max_completion_tokens"] == 16_384 + assert kwargs["timeout"] == 300 + assert "response_format" not in kwargs + for reasoning_key in ("reasoning", "reasoning_effort", "thinking"): + assert reasoning_key not in kwargs + + def test_claude_haiku_5_5_reasons_by_default_but_not_under_the_forced_tool(): """Haiku 5.5 thinks when the request omits the thinking parameter, so it is a thinking-default Claude: provider-default thinking, the thinking diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index a1cf6e4a..6f1b57bd 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -78,6 +78,8 @@ "openrouter/z-ai/glm-5.3": ("tool", None, 2400, 16_384), "openrouter/meta/muse-wonder": ("json", None, 300, 16_384), "gemini/gemini-4-argon": ("tool", None, 300, 16_384), + "gemini/gemini-4-carbon": ("tool", None, 300, 16_384), + "gemini/gemini-4-hydrogen": ("tool", None, 300, 16_384), } @@ -176,6 +178,8 @@ def test_provider_max_clamps_initial_completion_budget(monkeypatch): ("gemini/gemini-3-flash-preview", 65_535), ("gemini/gemini-3.6-flash", 65_536), ("gemini/gemini-4-argon", 32_768), + ("gemini/gemini-4-carbon", 32_768), + ("gemini/gemini-4-hydrogen", 65_536), ], ) def test_roster_model_provider_maximums_limit_escalation(model_id, expected): diff --git a/tests/test_paper_results.py b/tests/test_paper_results.py index a0f154d9..57ef7a1f 100644 --- a/tests/test_paper_results.py +++ b/tests/test_paper_results.py @@ -111,6 +111,23 @@ def test_gemini_4_argon_release_date_is_its_public_day(): assert MODEL_RELEASE_DATES["gemini-4-argon"] == "2026-10-13" +def test_gemini_4_carbon_release_date_is_its_public_day(): + """The Gemini API changelog lists Gemini 4 Carbon as generally available + on 2026-11-10, and Google's post of that day opens the API to all paying + developers and enterprises. The post says Google had released the model + two weeks earlier to cybersecurity defenders through its Fairwind + Program; that partner-only period is excluded under the + public-availability rule, as Gemini 4 Argon's is.""" + assert MODEL_RELEASE_DATES["gemini-4-carbon"] == "2026-11-10" + + +def test_gemini_4_hydrogen_release_date_is_recorded(): + """The Gemini API changelog lists Gemini 4 Hydrogen as generally + available on 2026-11-12, on the free and the paid tier, and Google's + launch post is dated the same day.""" + assert MODEL_RELEASE_DATES["gemini-4-hydrogen"] == "2026-11-12" + + def test_gpt_6_1_terra_release_date_is_its_announcement_day(): """OpenAI announced GPT-6.1 Terra on 2026-10-26 and opened it in ChatGPT (Plus, Pro, Business, Enterprise and Edu), Codex and the API (id From 1daeb7cf4896897e9312e3909fdaf5ce214e46af Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 13 Nov 2026 13:46:12 -0500 Subject: [PATCH 2/7] Rest the Gemini 4 Carbon and Hydrogen ceilings on the Models API and date the Carbon rate Wording fixes to the Gemini 4 Carbon and Gemini 4 Hydrogen cards and the Carbon price comment. No card field, price, release date or test assertion changes. - Ceiling: 32,768 completion tokens for Carbon and 65,536 for Hydrogen are the Models API's outputTokenLimit for each model, whose entry lists generateContent, the method the harness calls once without streaming. Google's long-decode and thinking guides describe the Interactions API and give the same per-request figures for non-streaming requests there; the card text and the previous commit's message cited them for generateContent. The Carbon card also says that the migration guide's examples use the Interactions API. - Price: the cost reconstruction takes no date, so the Carbon override prices a request at $2 / $10 per 1M and $0.10 cached input whenever it is made. Those are Google's rates only for requests made through 2026-11-30 (they double on 2026-12-01). The comment had said the row is priced at the rate in force on its run day and that the override keeps it priced if the price map changes on 2026-12-01. - Module docstring: cards encode serving-stack facts discovered during onboarding, and a card written before its model's onboarding says so in its notes, as these two do. Co-Authored-By: Claude Fable 5.2 --- policybench/config.py | 17 +++++---- policybench/model_cards.py | 73 ++++++++++++++++++++++--------------- tests/test_eval_no_tools.py | 7 ++-- 3 files changed, 56 insertions(+), 41 deletions(-) diff --git a/policybench/config.py b/policybench/config.py index 43b30dda..b88f86aa 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -265,14 +265,15 @@ # thinking tokens) and $0.10 cached input, the standard-tier rates Google # lists "through November 30, 2026"; from 2026-12-01 they double to # $4 / $20 and $0.20 (aistudio.google.com/docs/pricing, read 2026-11-13). - # The row is priced at the rate in force on its run day, as - # gemini-4-argon is at the same dated rates; this entry holds the rate - # through 2026-11-30. The page lists one rate at every prompt length, - # implicit caching only, and no free tier for this model. litellm - # 1.95.0's bundled price map has no gemini-4 id, and its remote map - # carries the same three introductory rates (read 2026-11-13); the - # override keeps the row priced if the map is unavailable or changes on - # 2026-12-01. + # A row is priced at the rate in force on its run day, as + # gemini-4-argon's is. The cost reconstruction takes no date, so this + # entry prices a request at these rates whenever it is made; they are + # Google's rates only for requests made through 2026-11-30. The page + # lists one rate at every prompt length, implicit caching only, and no + # free tier for this model. litellm 1.95.0's bundled price map has no + # gemini-4 id, and its remote map carries the same three introductory + # rates (read 2026-11-13); the override prices a request from the rates + # above whether or not the map is available. "gemini-4-carbon": {"input": 2.0, "output": 10.0, "cache_read": 0.10}, # gemini-4-hydrogen: $1.50 / $7.50 per 1M input/output tokens (output # includes thinking tokens) and $0.075 cached input, the paid tier's diff --git a/policybench/model_cards.py b/policybench/model_cards.py index 3e7531ea..8e740c6a 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -7,7 +7,8 @@ completion budget. None of this is model *configuration* — every model still runs unconfigured at its provider-default reasoning effort. Cards only encode serving-stack facts discovered during onboarding (see -``policybench onboard``). +``policybench onboard``); a card written before its model's onboarding says +so in its notes. Cards OVERRIDE the family-prefix heuristics in ``eval_no_tools``; models without a card (or with a field left ``None``) keep the heuristic treatment. @@ -574,10 +575,11 @@ class ModelCard: # 2026-11-13, and from Gemini 4 Argon's card above. It records no request # to the model. Field by field: # - answer_contract "tool": Argon's contract, the board's forced answer - # tool. Google's migration guide says that upgrading from - # gemini-4-argon to gemini-4-carbon "requires no request schema - # changes" (aistudio.google.com/docs/whats-new-gemini-4-carbon), and - # litellm sends the forced call as it does for Argon (mode ANY with + # tool. Google's migration guide, whose examples use the Interactions + # API, says that upgrading from gemini-4-argon to gemini-4-carbon + # "requires no request schema changes" + # (aistudio.google.com/docs/whats-new-gemini-4-carbon), and litellm + # sends the forced call as it does for Argon (mode ANY with # allowed_function_names). # - explanation_chunk_size None: the canonical whole-scenario request # (chunking is closed to new models). @@ -587,15 +589,18 @@ class ModelCard: # in the thinking-default class and takes that class's 300s timeout. # The 16,384-token starting budget is the Gemini family's. # - request_timeout_seconds None: the class default, as on Argon. - # - provider_max_completion_tokens 32_768: what one non-streaming request - # returns. Google's long-decode guide gives 32K per non-streaming + # - provider_max_completion_tokens 32_768: the Models API's + # outputTokenLimit for the model, whose entry lists generateContent + # among its supported methods (models.get, read 2026-11-13). litellm + # makes one non-streaming generateContent request. Google's long-decode + # and thinking guides describe the Interactions API and give the same + # figure for a non-streaming request there: 32K per non-streaming # request on gemini-4-carbon and gemini-4-argon - # (aistudio.google.com/docs/long-decode), its thinking guide gives - # 32,768 tokens for the same calls, and the Models API's - # outputTokenLimit for the model is 32,768 (models.get, read - # 2026-11-13). The model page lists 65,536 as "default synchronous", - # which the thinking guide gives for streaming calls on this model. - # litellm makes one non-streaming REST request. + # (aistudio.google.com/docs/long-decode) and 32,768 tokens for + # non-streaming calls on the two models + # (aistudio.google.com/docs/generate-content/thinking). The model page + # lists 65,536 as "default synchronous", which the thinking guide gives + # for streaming calls on this model. # - expected_cost_per_scenario_usd is not set: no per-scenario cost has # been measured. "gemini/gemini-4-carbon": ModelCard( @@ -615,13 +620,16 @@ class ModelCard: "gemini-4-argon to gemini-4-carbon needs no change to the " "request schema. The harness sends no thinkingConfig, and Google's " "thinking guide lists the model's default as On (medium), with " - "levels low, medium and high. That guide and the long-decode " - "guide give 32,768 tokens for one non-streaming request on this " - "model, which is what the harness sends, and the Models API's " - "outputTokenLimit is 32,768; the model page lists 65,536 as the " - "default synchronous output. Google's guide says Carbon spends " - "more thinking tokens per turn than Argon. Google lists $2/$10 " - "per 1M through 2026-11-30 and $4/$20 from 2026-12-01." + "levels low, medium and high. The Models API gives the model an " + "outputTokenLimit of 32,768 and lists generateContent, the method " + "the harness calls once without streaming, among its supported " + "methods. The thinking guide and the long-decode guide describe " + "the Interactions API and give the same 32,768 tokens for one " + "non-streaming request on this model; the model page lists " + "65,536 as the default synchronous output. Google's migration " + "guide says Carbon spends more thinking tokens per turn than " + "Argon on complex tasks. Google lists $2/$10 per 1M through " + "2026-11-30 and $4/$20 from 2026-12-01." ), ), # Gemini 4 Hydrogen. Like the Carbon card above, this card is written @@ -641,12 +649,14 @@ class ModelCard: # sends no thinkingConfig, so this model's default is high where # Carbon's and Argon's is medium. # - request_timeout_seconds None: the class default. - # - provider_max_completion_tokens 65_536: what one non-streaming request - # returns. Google's long-decode and thinking guides give 65,536 tokens - # for non-streaming gemini-4-hydrogen calls - # (aistudio.google.com/docs/long-decode), and the Models API's - # outputTokenLimit for the model is 65,536 (models.get, read - # 2026-11-13). + # - provider_max_completion_tokens 65_536: the Models API's + # outputTokenLimit for the model, whose entry lists generateContent + # among its supported methods (models.get, read 2026-11-13). litellm + # makes one non-streaming generateContent request. Google's long-decode + # and thinking guides describe the Interactions API and give the same + # 65,536 tokens for non-streaming gemini-4-hydrogen calls there + # (aistudio.google.com/docs/long-decode, + # aistudio.google.com/docs/generate-content/thinking). # - expected_cost_per_scenario_usd is not set: no per-scenario cost has # been measured. "gemini/gemini-4-hydrogen": ModelCard( @@ -665,10 +675,13 @@ class ModelCard: "sends no thinkingConfig, and Google's thinking guide lists the " "model's default as On (high), with levels low, medium, high " "and xhigh, where Gemini 4 Carbon's and Argon's default is " - "medium. That guide and the long-decode guide give 65,536 " - "tokens for one non-streaming request on this model, and the " - "Models API's outputTokenLimit is 65,536. Google lists $1.50/" - "$7.50 per 1M." + "medium. The Models API gives the model an outputTokenLimit of " + "65,536 and lists generateContent, the method the harness calls " + "once without streaming, among its supported methods; the " + "thinking guide and the long-decode guide describe the " + "Interactions API and give the same 65,536 tokens for one " + "non-streaming request on this model. Google lists $1.50/$7.50 " + "per 1M." ), ), "openrouter/z-ai/glm-5.3": ModelCard( diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index c9247eb1..6ab48733 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -3359,9 +3359,10 @@ def test_gemini_4_hydrogen_override_reconstructs_token_costs( assert reconstructed.is_estimated is False -# Google's listed per-1M rates (input, cached input, output), written out -# here and not read from PRICE_OVERRIDES_PER_1M, so the property below checks -# the override against the page's figures. +# Google's listed per-1M rates (input, cached input, output; Carbon's are the +# rates the page lists through 2026-11-30), written out here and not read from +# PRICE_OVERRIDES_PER_1M, so the property below checks the override against +# the page's figures. GEMINI_4_LISTED_RATES_PER_1M = { "gemini-4-carbon": (2.0, 0.10, 10.0), "gemini-4-hydrogen": (1.5, 0.075, 7.5), From 01d822ab345664f1608ec51a18f137cce8f3adf3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 14 Nov 2026 17:55:37 -0500 Subject: [PATCH 3/7] Rewrite the Gemini 4 Carbon and Hydrogen cards on their onboarding: Carbon answers as JSON The two cards were written from Google's pages before any request to either model. Both models are now onboarded (2026-11-14), and the cards say what the onboarding found, on Gemini 4 Argon's pattern. No price, release date or ceiling changes, and no board result. - Carbon contract: JSON. The gauntlet, run with no card, sent the forced answer tool first; the API accepted the forced call, but the model reasoned until the 16,384-token budget ran out (16,381 reasoning tokens, finish=length, 0 of 3 outputs parsed, 203s). The JSON contract then passed 3/3 (9,964 tokens, 105s) and 16/16 whole-scenario (7,257 tokens, 79s). The procedure fixed before the runs makes the card JSON when the tool probe fails, and the tool contract was not re-tested. Argon's tool_choice auto rule applies only to a forced-tool row, so Carbon has no auto sensitivity run. - Hydrogen contract: the forced tool, as before. The gauntlet passed 3/3 (10,956 tokens, 48s) and 16/16 (11,947 tokens, 51s). The notes hold one placeholder sentence, marked HPROBE-RESULT, for the harness-path probe's forced-against-auto reasoning result, which is still running. - Timeouts: Carbon 900s. Google recommends client timeouts of at least 15 minutes, and a retry at the 32,768-token ceiling takes about 345 to 410s at the gauntlet's 80 to 95 tokens a second. Hydrogen 600s: a request at its 65,536-token ceiling takes about 280 to 290s at 228 to 234 tokens a second. - Expected cost per scenario: Carbon 0.075, Hydrogen 0.094, each the gauntlet's whole-scenario probe. Carbon's cost-reconstruction row uses that call's exact counts (1,271 prompt, 7,257 completion, $0.075112). Hydrogen's uses 11,947 completion tokens and Argon's 2,762-token prompt for the same request, marked HPROBE-PROMPT-TOKENS until Hydrogen's own count is in. - Reasoning: both models reasoned on all 50 calls of a one-question generateContent probe, in all five request shapes (Argon reasoned on none with JSON or no tools). generateContent reports usageMetadata.thoughtsTokenCount on every response, litellm 1.95.0 maps it to reasoning_tokens, and the harness records it. Every response names the requested model with no dated snapshot. - Ceilings: unchanged (Carbon 32,768, Hydrogen 65,536). Google's generateContent long-decode guide, read 2026-11-14, now backs them for the request the harness sends, one direct non-streaming REST call. The 16,384-token starting budget comes from code, not from the card. - Module docstring: back to main's sentence, since no card is written before onboarding any more. - Tests: Carbon's and Hydrogen's treatments in EXPECTED; card cost locks; the shared request test split so Carbon's case checks the JSON request and Hydrogen's the forced tool. Co-Authored-By: Claude Opus 5.5 --- policybench/model_cards.py | 265 +++++++++++++++++++++++------------- tests/test_eval_no_tools.py | 70 ++++++++-- tests/test_model_cards.py | 21 ++- 3 files changed, 249 insertions(+), 107 deletions(-) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index 8e740c6a..124a7b85 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -7,8 +7,7 @@ completion budget. None of this is model *configuration* — every model still runs unconfigured at its provider-default reasoning effort. Cards only encode serving-stack facts discovered during onboarding (see -``policybench onboard``); a card written before its model's onboarding says -so in its notes. +``policybench onboard``). Cards OVERRIDE the family-prefix heuristics in ``eval_no_tools``; models without a card (or with a field left ``None``) keep the heuristic treatment. @@ -571,117 +570,201 @@ class ModelCard: "shows only that the parameter is accepted." ), ), - # Gemini 4 Carbon. This card is written from Google's documentation, read - # 2026-11-13, and from Gemini 4 Argon's card above. It records no request - # to the model. Field by field: - # - answer_contract "tool": Argon's contract, the board's forced answer - # tool. Google's migration guide, whose examples use the Interactions - # API, says that upgrading from gemini-4-argon to gemini-4-carbon - # "requires no request schema changes" - # (aistudio.google.com/docs/whats-new-gemini-4-carbon), and litellm - # sends the forced call as it does for Argon (mode ANY with - # allowed_function_names). - # - explanation_chunk_size None: the canonical whole-scenario request - # (chunking is closed to new models). + # Gemini 4 Carbon. Every field below comes from the 2026-11-14 onboarding + # record in ~/reviews/policybench-gemini-4-carbon/{probes,onboard} (a raw + # generateContent probe, a one-question reasoning probe of 10 calls a + # shape, and the gauntlet report, run with no card from a checkout of + # origin/main; the gauntlet's exact token counts are in + # onboard/carbon/usage-rows.jsonl) and from Google's pages: + # - answer_contract "json". The gauntlet's 3-variable probe sent the + # board's forced answer tool (litellm sends mode ANY with + # allowed_function_names), and the API accepted the forced call, but the + # model reasoned until the 16,384-token budget ran out: 16,384 + # completion tokens, 16,381 of them reasoning, finish=length, 0 of 3 + # outputs parsed, 203s. The gauntlet then passed on the JSON contract, + # 3/3 (9,964 completion tokens, 105s) and 16/16 whole-scenario (7,257 + # tokens, 79s), and suggested "json". The onboarding procedure, written + # before the runs, makes the card JSON when the tool probe fails, and the + # tool contract was not re-tested: re-running the gauntlet until the tool + # probe passes would choose the contract on a retry. So the request + # declares no tool and asks for a JSON object, as the older Gemini rows' + # requests do by the family default. + # - explanation_chunk_size None: 16/16 on the canonical whole-scenario + # request (chunking is closed to new models anyway). # - thinking_budget True: the model reasons by default. Google's thinking # guide lists its default as "On (medium)", with levels low, medium and - # high (aistudio.google.com/docs/generate-content/thinking), so it is - # in the thinking-default class and takes that class's 300s timeout. - # The 16,384-token starting budget is the Gemini family's. - # - request_timeout_seconds None: the class default, as on Argon. + # high (aistudio.google.com/docs/generate-content/thinking, read + # 2026-11-13), and all 51 raw generateContent responses in the record + # carry thought tokens. The 16,384-token starting budget is not the + # card's: the harness gives every gemini/ id 16,384 + # (GEMINI_JSON_MAX_COMPLETION_TOKENS). The gauntlet used all of it on the + # forced-tool probe, 61% on the 3-variable JSON probe and 44% on the + # whole-scenario JSON probe (22% of the 32,768-token ceiling). + # - request_timeout_seconds 900. Google's Gemini 4 Carbon guide says hard + # prompts "can take several minutes" and recommends HTTP client + # timeouts of "at least 15 minutes" + # (aistudio.google.com/docs/whats-new-gemini-4-carbon, read 2026-11-13). + # The gauntlet's probes ran at about 80 to 95 completion tokens a + # second, so a first request at the 16,384-token budget takes about 170 + # to 205s and a retry at the 32,768-token ceiling about 345 to 410s, + # past the thinking class's 300s. Google's generateContent long-decode + # guide says the server closes a non-streaming connection after 10 + # minutes and recommends a 20-minute client read timeout + # (aistudio.google.com/docs/long-decode?generate-content=true, read + # 2026-11-14). # - provider_max_completion_tokens 32_768: the Models API's # outputTokenLimit for the model, whose entry lists generateContent - # among its supported methods (models.get, read 2026-11-13). litellm - # makes one non-streaming generateContent request. Google's long-decode - # and thinking guides describe the Interactions API and give the same - # figure for a non-streaming request there: 32K per non-streaming - # request on gemini-4-carbon and gemini-4-argon - # (aistudio.google.com/docs/long-decode) and 32,768 tokens for - # non-streaming calls on the two models - # (aistudio.google.com/docs/generate-content/thinking). The model page - # lists 65,536 as "default synchronous", which the thinking guide gives - # for streaming calls on this model. - # - expected_cost_per_scenario_usd is not set: no per-scenario cost has - # been measured. + # among its supported methods (models.get on + # generativelanguage.googleapis.com/v1beta/models/gemini-4-carbon, read + # 2026-11-13). Google's generateContent long-decode guide gives the same + # cap for the request the harness sends, one direct non-streaming REST + # generateContent call: such calls on gemini-4-carbon "Default to 32K + # (32,768) output tokens", and a larger max_output_tokens without a + # continuation_token returns a 400 error + # (aistudio.google.com/docs/long-decode?generate-content=true, read + # 2026-11-14). The model page lists 65,536 as "default synchronous", + # which that guide gives for streaming and SDK calls. + # - expected_cost_per_scenario_usd 0.075: the gauntlet's whole-scenario + # probe, 1,271 prompt and 7,257 completion tokens, $0.075112 at $2/$10 + # per 1M, Google's rate through 2026-11-30. "gemini/gemini-4-carbon": ModelCard( litellm_id="gemini/gemini-4-carbon", - answer_contract="tool", + answer_contract="json", + request_timeout_seconds=900, thinking_budget=True, provider_max_completion_tokens=32_768, + expected_cost_per_scenario_usd=0.075, notes=( - "Written 2026-11-13 from Google's documentation, read that day; " - "the card records no request to the model. Google opened the " - "model to the public on 2026-11-10 (the Gemini API changelog " - "lists gemini-4-carbon as generally available that day; an " - "earlier release went to cybersecurity defenders through its " - "Fairwind Program). The card takes Gemini 4 Argon's treatment, " - "the board's forced answer tool on the whole-scenario request: " - "Google's migration guide says that upgrading from " - "gemini-4-argon to gemini-4-carbon needs no change to the " - "request schema. The harness sends no thinkingConfig, and Google's " - "thinking guide lists the model's default as On (medium), with " - "levels low, medium and high. The Models API gives the model an " - "outputTokenLimit of 32,768 and lists generateContent, the method " - "the harness calls once without streaming, among its supported " - "methods. The thinking guide and the long-decode guide describe " - "the Interactions API and give the same 32,768 tokens for one " - "non-streaming request on this model; the model page lists " - "65,536 as the default synchronous output. Google's migration " - "guide says Carbon spends more thinking tokens per turn than " - "Argon on complex tasks. Google lists $2/$10 per 1M through " + "Onboarded 2026-11-14. Google opened the model to the public on " + "2026-11-10 (the Gemini API changelog lists gemini-4-carbon as " + "generally available that day; an earlier release went to " + "cybersecurity defenders through its Fairwind Program). The row " + "answers on the JSON contract. The gauntlet, run with no card, " + "sent the board's forced answer tool first, and the Gemini API " + "accepted the forced call, but on the 3-variable probe the model " + "reasoned until the 16,384-token budget ran out (16,381 reasoning " + "tokens, 0 of 3 outputs parsed, 203s). The gauntlet then passed " + "on the JSON contract, 3/3 (9,964 completion tokens, 105s) and " + "16/16 whole-scenario (7,257 tokens, 79s), about $0.075 per " + "scenario at $2/$10 per 1M; the tool contract was not re-tested. " + "Gemini 4 Argon's rule, a tool_choice auto sensitivity run only " + "if forcing the tool suppresses reasoning, applies only to a row " + "that answers through the forced tool, so this row has no such " + "run. The harness sends no thinkingConfig, and Google's thinking " + "guide lists the model's default as On (medium). Unlike Argon, " + "the model reasoned on every call of a one-question " + "generateContent probe (10 calls a shape): thought tokens on all " + "50 calls, under a forced function call, tool_choice auto, mode " + "ANY with no named function, a JSON response type and no tools " + "(Argon on the same probe: 6 forced, 7 auto, 5 with mode ANY and " + "no named function, and none with a JSON response type or no " + "tools). On that one-question request, unlike the gauntlet's, " + "the forced calls returned a function call 10 times out of 10, " + "with a median of 2,515 thought tokens against 2,988 under auto. " + "generateContent reports the count as " + "usageMetadata.thoughtsTokenCount (on all 51 raw responses), " + "litellm 1.95.0 maps it to reasoning_tokens and counts it inside " + "completion_tokens, and the harness records it as each row's " + "reasoning_tokens (16,381, 9,318 and 6,056 on the three gauntlet " + "calls). Every response names the model as gemini-4-carbon, the " + "requested name, with no dated snapshot (modelVersion on the 51 " + "raw responses). The Models API gives an outputTokenLimit of " + "32,768, and Google's generateContent long-decode guide gives the " + "same 32,768 tokens for one direct non-streaming REST call, which " + "is what the harness sends; the model page lists 65,536 as the " + "default synchronous output, which the guide gives for streaming " + "and SDK calls. The timeout is 900s: Google recommends client " + "timeouts of at least 15 minutes, and a retry at the 32,768-token " + "ceiling takes about 345 to 410s at the gauntlet's 80 to 95 " + "completion tokens a second. Google lists $2/$10 per 1M through " "2026-11-30 and $4/$20 from 2026-12-01." ), ), - # Gemini 4 Hydrogen. Like the Carbon card above, this card is written - # from Google's documentation, read 2026-11-13, and records no request to - # the model. Field by field: - # - answer_contract "tool": the board's forced answer tool, the contract - # of every Gemini card from Gemini 3.6 Flash on. Google's model page - # lists function calling as supported - # (aistudio.google.com/docs/models/gemini-4-hydrogen). - # - explanation_chunk_size None: the canonical whole-scenario request - # (chunking is closed to new models). + # Gemini 4 Hydrogen. Every field below comes from the 2026-11-14 + # onboarding record in ~/reviews/policybench-gemini-4-carbon/{probes,onboard} + # (a raw generateContent probe, a one-question reasoning probe of 10 calls + # a shape, and the gauntlet report, run with no card from a checkout of + # origin/main) and from Google's pages: + # - answer_contract "tool": the Gemini API accepts a forced function call + # on this model (litellm sends mode ANY with allowed_function_names), + # and the gauntlet passed on it, 3/3 (10,956 completion tokens, 48s) and + # 16/16 whole-scenario (11,947 tokens, 51s), finish=tool_calls, so the + # row takes the board's forced answer tool, as every Gemini card from + # Gemini 3.6 Flash on does except Gemini 4 Carbon's. + # - explanation_chunk_size None: 16/16 on the canonical whole-scenario + # request (chunking is closed to new models anyway). # - thinking_budget True: the model reasons by default. Google's thinking # guide lists its default as "On (high)", with levels low, medium, high - # and xhigh (aistudio.google.com/docs/generate-content/thinking), so it - # is in the thinking-default class and takes that class's 300s timeout. - # The 16,384-token starting budget is the Gemini family's. The harness - # sends no thinkingConfig, so this model's default is high where - # Carbon's and Argon's is medium. - # - request_timeout_seconds None: the class default. + # and xhigh (aistudio.google.com/docs/generate-content/thinking, read + # 2026-11-13), and all 51 raw generateContent responses in the record + # carry thought tokens. The harness sends no thinkingConfig, so this + # model's default is high where Carbon's and Argon's is medium. The + # 16,384-token starting budget is not the card's: the harness gives every + # gemini/ id 16,384 (GEMINI_JSON_MAX_COMPLETION_TOKENS). The gauntlet + # used 67% of it on the 3-variable probe and 73% on the whole-scenario + # probe (18% of the 65,536-token ceiling). + # - request_timeout_seconds 600. At the gauntlet's 228 to 234 completion + # tokens a second, a request at the 65,536-token ceiling (the second + # budget escalation) takes about 280 to 290s, which leaves no margin + # under the thinking class's 300s. Google's generateContent long-decode + # guide says the server closes a non-streaming connection after 10 + # minutes (aistudio.google.com/docs/long-decode?generate-content=true, + # read 2026-11-14). # - provider_max_completion_tokens 65_536: the Models API's # outputTokenLimit for the model, whose entry lists generateContent - # among its supported methods (models.get, read 2026-11-13). litellm - # makes one non-streaming generateContent request. Google's long-decode - # and thinking guides describe the Interactions API and give the same - # 65,536 tokens for non-streaming gemini-4-hydrogen calls there - # (aistudio.google.com/docs/long-decode, - # aistudio.google.com/docs/generate-content/thinking). - # - expected_cost_per_scenario_usd is not set: no per-scenario cost has - # been measured. + # among its supported methods (models.get on + # generativelanguage.googleapis.com/v1beta/models/gemini-4-hydrogen, read + # 2026-11-13). Google's generateContent long-decode guide gives the same + # cap for every call on this model: "all gemini-4-hydrogen calls" + # default to 64K (65,536) output tokens, streaming and non-streaming + # (aistudio.google.com/docs/long-decode?generate-content=true, read + # 2026-11-14). + # - expected_cost_per_scenario_usd 0.094: the gauntlet's figure for its + # whole-scenario probe, 11,947 completion tokens at $7.50 per 1M and the + # prompt at $1.50 (the report gives the cost to three places and no + # prompt count). "gemini/gemini-4-hydrogen": ModelCard( litellm_id="gemini/gemini-4-hydrogen", answer_contract="tool", + request_timeout_seconds=600, thinking_budget=True, provider_max_completion_tokens=65_536, + expected_cost_per_scenario_usd=0.094, notes=( - "Written 2026-11-13 from Google's documentation, read that day; " - "the card records no request to the model. The Gemini API " - "changelog lists gemini-4-hydrogen as generally available on " - "2026-11-12, on the free and the paid tier. The card takes the " - "board's forced answer tool on the whole-scenario request, as " - "every Gemini card from Gemini 3.6 Flash on does; Google's " - "model page lists function calling as supported. The harness " - "sends no thinkingConfig, and Google's thinking guide lists the " - "model's default as On (high), with levels low, medium, high " - "and xhigh, where Gemini 4 Carbon's and Argon's default is " - "medium. The Models API gives the model an outputTokenLimit of " - "65,536 and lists generateContent, the method the harness calls " - "once without streaming, among its supported methods; the " - "thinking guide and the long-decode guide describe the " - "Interactions API and give the same 65,536 tokens for one " - "non-streaming request on this model. Google lists $1.50/$7.50 " - "per 1M." + "Onboarded 2026-11-14. The Gemini API changelog lists " + "gemini-4-hydrogen as generally available on 2026-11-12, on the " + "free and the paid tier. The Gemini API accepts a forced function " + "call on this model, so the row takes the board's forced answer " + "tool: the gauntlet, run with no card, passed 3/3 (10,956 " + "completion tokens, 48s) and 16/16 whole-scenario (11,947 tokens, " + "51s) on the tool contract, about $0.094 per scenario at " + "$1.50/$7.50 per 1M. The harness sends no thinkingConfig, and " + "Google's thinking guide lists the model's default as On (high), " + "where Gemini 4 Carbon's and Argon's is medium. Unlike Argon, the " + "model reasoned on every call of a one-question generateContent " + "probe (10 calls a shape): thought tokens on all 50 calls, under " + "a forced function call, tool_choice auto, mode ANY with no named " + "function, a JSON response type and no tools, with a median of " + "5,273 thought tokens forced against 5,496 under auto, and the " + "forced calls returned a function call 10 times out of 10. " + "HPROBE-RESULT: the harness-path probe's forced-against-auto " + "comparison (the board's own request on the frozen manifest's " + "scenarios 0 to 15, 16 calls a shape, forced, tool_choice auto " + "and JSON, under a rule fixed before its first call) goes in this " + "sentence. generateContent reports the count as " + "usageMetadata.thoughtsTokenCount (on all 51 raw responses), and " + "litellm 1.95.0 maps it to reasoning_tokens and counts it inside " + "completion_tokens (checked on the saved responses), which the " + "harness records as each row's reasoning_tokens. Every response " + "names the model as gemini-4-hydrogen, the requested name, with " + "no dated snapshot (modelVersion on the 51 raw responses). The " + "Models API gives an outputTokenLimit of 65,536, and Google's " + "generateContent long-decode guide gives the same 65,536 tokens " + "for every call on this model, streaming or not. The gauntlet " + "used 67% and 73% of the 16,384-token starting budget. The " + "timeout is 600s: a request at the 65,536-token ceiling takes " + "about 280 to 290s at the gauntlet's 228 to 234 completion tokens " + "a second. Google lists $1.50/$7.50 per 1M." ), ), "openrouter/z-ai/glm-5.3": ModelCard( diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index 6ab48733..b133e9ad 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -3304,6 +3304,12 @@ def test_gemini_4_carbon_runs_first_party_at_googles_dated_rate(): (1_000_000, 1_000_000, 0, 12.0), (1_000_000, 0, 1_000_000, 0.10), (1_000_000, 0, 250_000, 1.525), + # The card-free gauntlet's whole-scenario probe (json-full): 1,271 + # prompt and 7,257 completion tokens (6,056 of them reasoning), no + # cached input, the counts the harness's usage extraction returned + # for that call. The gauntlet priced it at $0.075112 from litellm's + # map at the same rates; that is the card's $0.075. + (1_271, 7_257, 0, 0.075112), ], ) def test_gemini_4_carbon_override_reconstructs_token_costs( @@ -3342,6 +3348,11 @@ def test_gemini_4_hydrogen_runs_first_party_at_googles_list_rate(): (1_000_000, 1_000_000, 0, 9.0), (1_000_000, 0, 1_000_000, 0.075), (1_000_000, 0, 250_000, 1.14375), + # The card-free gauntlet's whole-scenario probe (tool-full) used + # 11,947 completion tokens. Its report gives no prompt count; 2,762 + # is Gemini 4 Argon's harness-path count for the same request, + # pending Hydrogen's own. That is the card's $0.094. + (2_762, 11_947, 0, 0.0937455), # HPROBE-PROMPT-TOKENS ], ) def test_gemini_4_hydrogen_override_reconstructs_token_costs( @@ -3599,28 +3610,59 @@ def test_gemini_4_argon_request_forces_the_answer_tool_and_sets_no_reasoning( assert reasoning_key not in kwargs -@pytest.mark.parametrize( - "model_id", ["gemini/gemini-4-carbon", "gemini/gemini-4-hydrogen"] -) -def test_gemini_4_carbon_and_hydrogen_force_the_answer_tool_and_set_no_reasoning( - mini_scenario, model_id +def test_gemini_4_carbon_request_asks_for_json_and_sets_no_reasoning(mini_scenario): + """Gemini 4 Carbon's card selects the JSON contract: in the card-free + gauntlet the API accepted the forced answer tool, but the model reasoned + until the 16,384-token budget ran out (0 of 3 outputs parsed), and the + JSON probes passed 3/3 and 16/16. So the request declares no tool and + asks for a JSON object. It carries no reasoning parameter, so the model + reasons at Google's default, medium per Google's thinking guide. It takes + the card's 900s timeout and the Gemini family's 16,384-token starting + budget.""" + from policybench.eval_no_tools import ( + _answer_contract_for_model, + _chat_completion_request_kwargs, + ) + + model_id = "gemini/gemini-4-carbon" + assert _answer_contract_for_model(model_id, env={}) == "json" + assert _thinking_configuration(model_id) == {"mode": "provider_default"} + assert _request_timeout_seconds(model_id) == 900 + + messages, kwargs = _chat_completion_request_kwargs( + mini_scenario, ["income_tax"], model_id + ) + + assert kwargs["response_format"] == {"type": "json_object"} + assert "tools" not in kwargs + assert "tool_choice" not in kwargs + assert "JSON object" in messages[0]["content"] + assert kwargs["max_completion_tokens"] == 16_384 + assert kwargs["timeout"] == 900 + for reasoning_key in ("reasoning", "reasoning_effort", "thinking"): + assert reasoning_key not in kwargs + + +def test_gemini_4_hydrogen_request_forces_the_answer_tool_and_sets_no_reasoning( + mini_scenario, ): - """Gemini 4 Carbon's and Gemini 4 Hydrogen's cards select the board's - forced answer tool, as Gemini 4 Argon's does (the older Gemini rows - answer as JSON by the family default). The request carries no reasoning - parameter, so each model reasons at Google's default, which Google's - thinking guide gives as medium for Carbon and high for Hydrogen. Each - takes the thinking-default class's 300s timeout and the Gemini family's - 16,384-token starting budget.""" + """The Gemini API accepts a forced function call on Gemini 4 Hydrogen and + the card-free gauntlet passed 3/3 and 16/16 on it, so its card selects + the board's forced answer tool, as Gemini 4 Argon's does (the older + Gemini rows answer as JSON by the family default). The request carries no + reasoning parameter, so the model reasons at Google's default, high per + Google's thinking guide. It takes the card's 600s timeout and the Gemini + family's 16,384-token starting budget.""" from policybench.eval_no_tools import ( ANSWER_FUNCTION_NAME, _answer_contract_for_model, _chat_completion_request_kwargs, ) + model_id = "gemini/gemini-4-hydrogen" assert _answer_contract_for_model(model_id, env={}) == "tool" assert _thinking_configuration(model_id) == {"mode": "provider_default"} - assert _request_timeout_seconds(model_id) == 300 + assert _request_timeout_seconds(model_id) == 600 _, kwargs = _chat_completion_request_kwargs(mini_scenario, ["income_tax"], model_id) @@ -3632,7 +3674,7 @@ def test_gemini_4_carbon_and_hydrogen_force_the_answer_tool_and_set_no_reasoning ANSWER_FUNCTION_NAME ] assert kwargs["max_completion_tokens"] == 16_384 - assert kwargs["timeout"] == 300 + assert kwargs["timeout"] == 600 assert "response_format" not in kwargs for reasoning_key in ("reasoning", "reasoning_effort", "thinking"): assert reasoning_key not in kwargs diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index 6f1b57bd..e6bf02ea 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -78,8 +78,8 @@ "openrouter/z-ai/glm-5.3": ("tool", None, 2400, 16_384), "openrouter/meta/muse-wonder": ("json", None, 300, 16_384), "gemini/gemini-4-argon": ("tool", None, 300, 16_384), - "gemini/gemini-4-carbon": ("tool", None, 300, 16_384), - "gemini/gemini-4-hydrogen": ("tool", None, 300, 16_384), + "gemini/gemini-4-carbon": ("json", None, 900, 16_384), + "gemini/gemini-4-hydrogen": ("tool", None, 600, 16_384), } @@ -140,6 +140,23 @@ def test_gpt_56_cards_record_measured_full_run_cost(model_id, expected_cost): assert card_for(model_id).expected_cost_per_scenario_usd == expected_cost +@pytest.mark.parametrize( + ("model_id", "expected_cost"), + [ + ("gemini/gemini-4-carbon", 0.075), + ("gemini/gemini-4-hydrogen", 0.094), + ], +) +def test_gemini_4_carbon_and_hydrogen_cards_record_the_gauntlet_cost( + model_id, expected_cost +): + """Each card carries its card-free gauntlet's whole-scenario cost, rounded + to three places as ``policybench onboard`` suggests it; the token counts + behind each figure are rows of the cost-reconstruction tests in + tests/test_eval_no_tools.py.""" + assert card_for(model_id).expected_cost_per_scenario_usd == expected_cost + + @pytest.mark.parametrize( ("provider_max", "expected"), [(30_000, 30_000), (200_000, 128_000)] ) From 325e6494e6c00f1c6b01c59f15740c8ff073d6de Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 14 Nov 2026 18:12:17 -0500 Subject: [PATCH 4/7] Record Gemini 4 Hydrogen's harness-path reasoning probe on its card The harness-path probe sent the board's own request for the frozen manifest's scenarios 0 to 15, 16 calls a shape, under the rule fixed before its first call. Calls with reasoning tokens: forced 16 of 16, tool_choice auto 16 of 16 and JSON 16 of 16; forced against auto, two-sided Fisher exact p = 1.0. Forcing the tool did not lower how often the model reasoned, so the rule adds no tool_choice auto sensitivity run. Hydrogen's notes now give that result, and the probe's length finishes: 6 forced, 6 auto and 2 JSON calls ran out of the 16,384-token budget on one request each, which the board escalates. The probe's forced call on scenario 0 is the same request as the gauntlet's tool-full probe (the same messages, answer tool and 16,384-token budget), and Google returned 2,762 prompt tokens for it. The cost-reconstruction row and the card's comment now give that count. With the gauntlet's 11,947 completion tokens at $1.50/$7.50 per 1M it prices at $0.0937455, so the card's 0.094 stands. Co-Authored-By: Claude Opus 5.5 --- policybench/model_cards.py | 37 ++++++++++++++++++++++++++----------- tests/test_eval_no_tools.py | 13 ++++++++----- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index 124a7b85..dc7c04d3 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -683,8 +683,10 @@ class ModelCard: # Gemini 4 Hydrogen. Every field below comes from the 2026-11-14 # onboarding record in ~/reviews/policybench-gemini-4-carbon/{probes,onboard} # (a raw generateContent probe, a one-question reasoning probe of 10 calls - # a shape, and the gauntlet report, run with no card from a checkout of - # origin/main) and from Google's pages: + # a shape, the gauntlet report, run with no card from a checkout of + # origin/main, and a harness-path probe of the board's own request, 16 + # calls a shape, run from an export of this card's branch) and from + # Google's pages: # - answer_contract "tool": the Gemini API accepts a forced function call # on this model (litellm sends mode ANY with allowed_function_names), # and the gauntlet passed on it, 3/3 (10,956 completion tokens, 48s) and @@ -719,10 +721,12 @@ class ModelCard: # default to 64K (65,536) output tokens, streaming and non-streaming # (aistudio.google.com/docs/long-decode?generate-content=true, read # 2026-11-14). - # - expected_cost_per_scenario_usd 0.094: the gauntlet's figure for its - # whole-scenario probe, 11,947 completion tokens at $7.50 per 1M and the - # prompt at $1.50 (the report gives the cost to three places and no - # prompt count). + # - expected_cost_per_scenario_usd 0.094: the gauntlet's whole-scenario + # probe, 2,762 prompt and 11,947 completion tokens, $0.0937455 at + # $1.50/$7.50 per 1M. The gauntlet report gives the cost to three places + # and no prompt count; the 2,762 is the prompt count Google returned for + # the harness-path probe's forced call on scenario 0, the same request + # (the same messages, answer tool and 16,384-token budget). "gemini/gemini-4-hydrogen": ModelCard( litellm_id="gemini/gemini-4-hydrogen", answer_contract="tool", @@ -747,11 +751,22 @@ class ModelCard: "function, a JSON response type and no tools, with a median of " "5,273 thought tokens forced against 5,496 under auto, and the " "forced calls returned a function call 10 times out of 10. " - "HPROBE-RESULT: the harness-path probe's forced-against-auto " - "comparison (the board's own request on the frozen manifest's " - "scenarios 0 to 15, 16 calls a shape, forced, tool_choice auto " - "and JSON, under a rule fixed before its first call) goes in this " - "sentence. generateContent reports the count as " + "On the board's own request for the frozen manifest's scenarios " + "0 to 15 (16 calls a shape, under a rule fixed before the first " + "call), the forced tool returned reasoning tokens on 16 of 16 " + "calls, tool_choice auto on 16 of 16 and the JSON contract on 16 " + "of 16 (forced against auto, two-sided Fisher exact p = 1.0), so " + "forcing the tool did not lower how often the model reasoned, and " + "the row has no tool_choice auto sensitivity run (Gemini 4 Argon's " + "rule adds one only if forcing the tool suppresses reasoning). " + "That probe sent each request once, with no budget escalation: 6 " + "forced and 6 auto calls, on the same six scenarios, and 2 JSON " + "calls ran out of the 16,384-token budget (finish=length, with " + "14,869 to 16,381 reasoning tokens each), so 184 of 334 outputs " + "parsed under each tool mode and 262 under JSON; on such a call " + "the board doubles the budget, up to the 65,536-token ceiling, " + "and asks again. " + "generateContent reports the thought-token count as " "usageMetadata.thoughtsTokenCount (on all 51 raw responses), and " "litellm 1.95.0 maps it to reasoning_tokens and counts it inside " "completion_tokens (checked on the saved responses), which the " diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index b133e9ad..7dfac707 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -3348,11 +3348,14 @@ def test_gemini_4_hydrogen_runs_first_party_at_googles_list_rate(): (1_000_000, 1_000_000, 0, 9.0), (1_000_000, 0, 1_000_000, 0.075), (1_000_000, 0, 250_000, 1.14375), - # The card-free gauntlet's whole-scenario probe (tool-full) used - # 11,947 completion tokens. Its report gives no prompt count; 2,762 - # is Gemini 4 Argon's harness-path count for the same request, - # pending Hydrogen's own. That is the card's $0.094. - (2_762, 11_947, 0, 0.0937455), # HPROBE-PROMPT-TOKENS + # The card-free gauntlet's whole-scenario probe (tool-full): 2,762 + # prompt and 11,947 completion tokens, priced with no cached input. + # The gauntlet's report gives no prompt count; 2,762 is the count + # Google returned for the harness-path probe's forced call on + # scenario 0, the same request (the same messages, answer tool and + # 16,384-token budget), which reported no cached input. That is the + # card's $0.094. + (2_762, 11_947, 0, 0.0937455), ], ) def test_gemini_4_hydrogen_override_reconstructs_token_costs( From e0062a9c2b5e5f991a398b2c3cabb47d8d29526a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 14 Nov 2026 19:08:59 -0500 Subject: [PATCH 5/7] State what bounds the Gemini 4 Carbon and Hydrogen requests, and tie the cards' cost to its counts Three readers checked the cards against the onboarding record and Google's saved pages and found every number true. This commit takes their points on what the comments and notes left out. - Timeouts. Google's generateContent long-decode guide says the server closes a non-streaming connection after 10 minutes, and its option table gives 20 minutes per request. Both cards now say what that means for their timeout: no request runs past 600s if the server closes at 10 minutes, and Carbon's 900s also covers the table's 20 minutes. Hydrogen's comment gives the slowest rates in the record (about 215 to 220 tokens a second), at which a request at the 65,536-token ceiling takes about 300s. - Carbon's budget. The harness doubles the 16,384-token budget once, to 32,768, and records any output still missing there as budget_exhausted_at_ceiling. The comment and the notes now say so. - Carbon's price. The override's comment now says to change the entry before any request made on or after 2026-12-01, and both price comments say what the record holds about litellm's remote map (the gauntlet's rates on 2026-11-14) in place of an unsaved read. - Hydrogen's notes name the harness-path probe's households as the frozen manifest's first 16 scenarios, and attach the reasoning-token range to all 14 calls that ran out of budget. - The medians of thought tokens that end in .5 read "about". - test_gemini_4_carbon_and_hydrogen_cards_record_the_gauntlet_cost now reconstructs each gauntlet probe's cost from its token counts at the price override and checks the card against it. Co-Authored-By: Claude Fable 5.2 --- policybench/config.py | 17 ++++++----- policybench/model_cards.py | 58 +++++++++++++++++++++++++------------- tests/test_model_cards.py | 26 ++++++++++++----- 3 files changed, 68 insertions(+), 33 deletions(-) diff --git a/policybench/config.py b/policybench/config.py index b88f86aa..e66ce0f3 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -265,14 +265,16 @@ # thinking tokens) and $0.10 cached input, the standard-tier rates Google # lists "through November 30, 2026"; from 2026-12-01 they double to # $4 / $20 and $0.20 (aistudio.google.com/docs/pricing, read 2026-11-13). - # A row is priced at the rate in force on its run day, as - # gemini-4-argon's is. The cost reconstruction takes no date, so this + # The board prices a row at the rate in force on its run day, as it + # does gemini-4-argon's. The cost reconstruction takes no date, so this # entry prices a request at these rates whenever it is made; they are - # Google's rates only for requests made through 2026-11-30. The page + # Google's rates only for requests made through 2026-11-30. Change this + # entry to 4.0 / 20.0 / 0.20 before any gemini-4-carbon request made on + # or after 2026-12-01. The page # lists one rate at every prompt length, implicit caching only, and no # free tier for this model. litellm 1.95.0's bundled price map has no - # gemini-4 id, and its remote map carries the same three introductory - # rates (read 2026-11-13); the override prices a request from the rates + # gemini-4 id, and its remote map priced the 2026-11-14 gauntlet's calls + # at the same $2 / $10; the override prices a request from the rates # above whether or not the map is available. "gemini-4-carbon": {"input": 2.0, "output": 10.0, "cache_read": 0.10}, # gemini-4-hydrogen: $1.50 / $7.50 per 1M input/output tokens (output @@ -281,8 +283,9 @@ # read 2026-11-13). The page lists one rate at every prompt length and # implicit caching only; it also lists a free tier, which the row's cost # does not use. litellm 1.95.0's bundled price map has no gemini-4 id, and - # its remote map carries the same three rates (read 2026-11-13); the - # override keeps the row priced if the map is unavailable. + # its remote map priced the 2026-11-14 gauntlet's calls at the same + # $1.50 / $7.50; the override prices a request from the rates above + # whether or not the map is available. "gemini-4-hydrogen": {"input": 1.5, "output": 7.5, "cache_read": 0.075}, # glm-5.3: $1.40 / $4.40 per 1M (OpenRouter list, 2026-09-03). "glm-5.3": {"input": 1.40, "output": 4.40}, diff --git a/policybench/model_cards.py b/policybench/model_cards.py index dc7c04d3..cc395ceb 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -609,9 +609,16 @@ class ModelCard: # to 205s and a retry at the 32,768-token ceiling about 345 to 410s, # past the thinking class's 300s. Google's generateContent long-decode # guide says the server closes a non-streaming connection after 10 - # minutes and recommends a 20-minute client read timeout + # minutes and recommends a 20-minute client read timeout, and its option + # table gives 20 minutes per request for a non-streaming loop # (aistudio.google.com/docs/long-decode?generate-content=true, read - # 2026-11-14). + # 2026-11-14). If the server closes at 10 minutes, no request on this + # row runs past 600s and the timeout's last 300s never apply; 900s also + # covers the table's 20 minutes. + # The harness doubles the 16,384-token budget once, to the ceiling + # below, and records any output still missing there as + # budget_exhausted_at_ceiling, so a household that needs more than + # 32,768 completion tokens cannot finish on this row. # - provider_max_completion_tokens 32_768: the Models API's # outputTokenLimit for the model, whose entry lists generateContent # among its supported methods (models.get on @@ -660,7 +667,8 @@ class ModelCard: "no named function, and none with a JSON response type or no " "tools). On that one-question request, unlike the gauntlet's, " "the forced calls returned a function call 10 times out of 10, " - "with a median of 2,515 thought tokens against 2,988 under auto. " + "with a median of about 2,515 thought tokens against about 2,988 " + "under auto. " "generateContent reports the count as " "usageMetadata.thoughtsTokenCount (on all 51 raw responses), " "litellm 1.95.0 maps it to reasoning_tokens and counts it inside " @@ -676,7 +684,12 @@ class ModelCard: "and SDK calls. The timeout is 900s: Google recommends client " "timeouts of at least 15 minutes, and a retry at the 32,768-token " "ceiling takes about 345 to 410s at the gauntlet's 80 to 95 " - "completion tokens a second. Google lists $2/$10 per 1M through " + "completion tokens a second. Google's generateContent guide says " + "the server closes a non-streaming call after 10 minutes, which " + "would end a request at 600s. The harness doubles the " + "16,384-token budget once, to the 32,768-token ceiling, and " + "records any output still missing there as " + "budget_exhausted_at_ceiling. Google lists $2/$10 per 1M through " "2026-11-30 and $4/$20 from 2026-12-01." ), ), @@ -685,8 +698,8 @@ class ModelCard: # (a raw generateContent probe, a one-question reasoning probe of 10 calls # a shape, the gauntlet report, run with no card from a checkout of # origin/main, and a harness-path probe of the board's own request, 16 - # calls a shape, run from an export of this card's branch) and from - # Google's pages: + # calls a shape, run from an export of this card's branch before its + # rebase onto main, with the same request code) and from Google's pages: # - answer_contract "tool": the Gemini API accepts a forced function call # on this model (litellm sends mode ANY with allowed_function_names), # and the gauntlet passed on it, 3/3 (10,956 completion tokens, 48s) and @@ -707,11 +720,15 @@ class ModelCard: # probe (18% of the 65,536-token ceiling). # - request_timeout_seconds 600. At the gauntlet's 228 to 234 completion # tokens a second, a request at the 65,536-token ceiling (the second - # budget escalation) takes about 280 to 290s, which leaves no margin - # under the thinking class's 300s. Google's generateContent long-decode - # guide says the server closes a non-streaming connection after 10 - # minutes (aistudio.google.com/docs/long-decode?generate-content=true, - # read 2026-11-14). + # budget escalation) takes about 280 to 290s, and at the slowest rates + # in the record (about 215 to 220 a second, in the one-question probe + # and the smoke) about 300s, the thinking class's timeout. Google's + # generateContent long-decode guide says the server closes a + # non-streaming connection after 10 minutes, though its option table + # gives 20 minutes per request + # (aistudio.google.com/docs/long-decode?generate-content=true, read + # 2026-11-14). If the server closes at 10 minutes, a client timeout + # above 600s could not help this request. # - provider_max_completion_tokens 65_536: the Models API's # outputTokenLimit for the model, whose entry lists generateContent # among its supported methods (models.get on @@ -749,10 +766,10 @@ class ModelCard: "probe (10 calls a shape): thought tokens on all 50 calls, under " "a forced function call, tool_choice auto, mode ANY with no named " "function, a JSON response type and no tools, with a median of " - "5,273 thought tokens forced against 5,496 under auto, and the " + "5,273 thought tokens forced against about 5,496 under auto, and the " "forced calls returned a function call 10 times out of 10. " - "On the board's own request for the frozen manifest's scenarios " - "0 to 15 (16 calls a shape, under a rule fixed before the first " + "On the board's own request for the frozen manifest's first 16 " + "scenarios (16 calls a shape, under a rule fixed before the first " "call), the forced tool returned reasoning tokens on 16 of 16 " "calls, tool_choice auto on 16 of 16 and the JSON contract on 16 " "of 16 (forced against auto, two-sided Fisher exact p = 1.0), so " @@ -760,10 +777,11 @@ class ModelCard: "the row has no tool_choice auto sensitivity run (Gemini 4 Argon's " "rule adds one only if forcing the tool suppresses reasoning). " "That probe sent each request once, with no budget escalation: 6 " - "forced and 6 auto calls, on the same six scenarios, and 2 JSON " - "calls ran out of the 16,384-token budget (finish=length, with " - "14,869 to 16,381 reasoning tokens each), so 184 of 334 outputs " - "parsed under each tool mode and 262 under JSON; on such a call " + "forced, 6 auto and 2 JSON calls ran out of the 16,384-token " + "budget (finish=length, 14,869 to 16,381 reasoning tokens each), " + "the forced and auto calls on the same six scenarios, so 184 of " + "334 outputs parsed under each tool mode and 262 under JSON; on " + "such a call " "the board doubles the budget, up to the 65,536-token ceiling, " "and asks again. " "generateContent reports the thought-token count as " @@ -779,7 +797,9 @@ class ModelCard: "used 67% and 73% of the 16,384-token starting budget. The " "timeout is 600s: a request at the 65,536-token ceiling takes " "about 280 to 290s at the gauntlet's 228 to 234 completion tokens " - "a second. Google lists $1.50/$7.50 per 1M." + "a second, close to the thinking class's 300s, and Google's " + "generateContent guide says the server closes a non-streaming " + "call after 10 minutes. Google lists $1.50/$7.50 per 1M." ), ), "openrouter/z-ai/glm-5.3": ModelCard( diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index e6bf02ea..dbbf04b2 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -13,6 +13,7 @@ _answer_contract_for_model, _completion_budget_ceiling, _completion_controls, + _reconstruct_token_cost, _request_timeout_seconds, _required_explanation_chunk_size, ) @@ -141,19 +142,30 @@ def test_gpt_56_cards_record_measured_full_run_cost(model_id, expected_cost): @pytest.mark.parametrize( - ("model_id", "expected_cost"), + ("model_name", "model_id", "prompt_tokens", "completion_tokens", "expected_cost"), [ - ("gemini/gemini-4-carbon", 0.075), - ("gemini/gemini-4-hydrogen", 0.094), + ("gemini-4-carbon", "gemini/gemini-4-carbon", 1_271, 7_257, 0.075), + ("gemini-4-hydrogen", "gemini/gemini-4-hydrogen", 2_762, 11_947, 0.094), ], ) def test_gemini_4_carbon_and_hydrogen_cards_record_the_gauntlet_cost( - model_id, expected_cost + model_name, model_id, prompt_tokens, completion_tokens, expected_cost ): """Each card carries its card-free gauntlet's whole-scenario cost, rounded - to three places as ``policybench onboard`` suggests it; the token counts - behind each figure are rows of the cost-reconstruction tests in - tests/test_eval_no_tools.py.""" + to three places as ``policybench onboard`` suggests it. + + The token counts are the gauntlet's whole-scenario probe's, and the cost is + their reconstruction at the model's price override, so the card, the + counts and the override cannot drift apart. + """ + reconstructed = _reconstruct_token_cost( + model_name=model_name, + model_id=model_id, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + ) + assert reconstructed.is_estimated is False + assert round(reconstructed.usd, 3) == expected_cost assert card_for(model_id).expected_cost_per_scenario_usd == expected_cost From 3b16074774f76565981d3f57d67439118c21a8c1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 14 Nov 2026 23:46:45 -0500 Subject: [PATCH 6/7] Correct Carbon's timeout comment, and state Google's budget advice and the code that prices a row late Carbon's card comment said its 900s timeout covers the 20 minutes in Google's option table; 900s is 15 minutes. The comment now says what 900s exceeds: the 345 to 410s the gauntlet's rates predict for a request at the ceiling, and the 620s the slowest one-question probe rate predicts. Both cards now give the long-decode guide's 20-minute read-timeout advice beside its 10-minute close. Carbon's card says the gauntlet's tool probe ran once at 16,384 tokens, half the minimum Google's guide recommends for the model. The price-override comment names the code that prices a row after Google served the response: batch collection, a disk-cache replay and the analysis fallback. Each card's cost comment adds what its smoke households cost. Co-Authored-By: Claude Fable 5.2 --- policybench/config.py | 10 +++- policybench/model_cards.py | 105 +++++++++++++++++++++++-------------- 2 files changed, 76 insertions(+), 39 deletions(-) diff --git a/policybench/config.py b/policybench/config.py index e66ce0f3..dee10e7a 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -270,7 +270,15 @@ # entry prices a request at these rates whenever it is made; they are # Google's rates only for requests made through 2026-11-30. Change this # entry to 4.0 / 20.0 / 0.20 before any gemini-4-carbon request made on - # or after 2026-12-01. The page + # or after 2026-12-01. The code prices some rows after Google served + # the response: the harness prices a batch result when it collects it + # (batch_eval.rows_from_unit and _batch_spend_record); it prices a + # response the LiteLLM disk cache replays at the replay + # (eval_no_tools._extract_usage_metadata; eval-no-tools enables the + # cache); and, for a model whose usage records no cost, + # analysis.model_cost_latency computes one from the current entry's + # input and output rates. Once the entry changes, each would put the + # December rate on a request made in November. The pricing page # lists one rate at every prompt length, implicit caching only, and no # free tier for this model. litellm 1.95.0's bundled price map has no # gemini-4 id, and its remote map priced the 2026-11-14 gauntlet's calls diff --git a/policybench/model_cards.py b/policybench/model_cards.py index cc395ceb..459cd74c 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -571,18 +571,24 @@ class ModelCard: ), ), # Gemini 4 Carbon. Every field below comes from the 2026-11-14 onboarding - # record in ~/reviews/policybench-gemini-4-carbon/{probes,onboard} (a raw - # generateContent probe, a one-question reasoning probe of 10 calls a - # shape, and the gauntlet report, run with no card from a checkout of - # origin/main; the gauntlet's exact token counts are in - # onboard/carbon/usage-rows.jsonl) and from Google's pages: + # record in ~/reviews/policybench-gemini-4-carbon/{probes,onboard,smoke} + # (a raw generateContent probe, a one-question reasoning probe of 10 calls + # a shape, the gauntlet report, run with no card from a checkout of + # origin/main, and a 2-household smoke run on this card; the gauntlet's + # exact token counts are in onboard/carbon/usage-rows.jsonl) and from + # Google's pages: # - answer_contract "json". The gauntlet's 3-variable probe sent the # board's forced answer tool (litellm sends mode ANY with # allowed_function_names), and the API accepted the forced call, but the # model reasoned until the 16,384-token budget ran out: 16,384 # completion tokens, 16,381 of them reasoning, finish=length, 0 of 3 - # outputs parsed, 203s. The gauntlet then passed on the JSON contract, - # 3/3 (9,964 completion tokens, 105s) and 16/16 whole-scenario (7,257 + # outputs parsed, 203s. The gauntlet sends each probe once, with no + # budget escalation, so the tool request ran at half the + # max_output_tokens Google's Gemini 4 Carbon guide recommends: "at + # least 32,768, since Gemini 4 Carbon can spend more of the limit on + # thinking" (aistudio.google.com/docs/whats-new-gemini-4-carbon, read + # 2026-11-13). The gauntlet then passed on the JSON contract, 3/3 + # (9,964 completion tokens, 105s) and 16/16 whole-scenario (7,257 # tokens, 79s), and suggested "json". The onboarding procedure, written # before the runs, makes the card JSON when the tool probe fails, and the # tool contract was not re-tested: re-running the gauntlet until the tool @@ -607,16 +613,21 @@ class ModelCard: # The gauntlet's probes ran at about 80 to 95 completion tokens a # second, so a first request at the 16,384-token budget takes about 170 # to 205s and a retry at the 32,768-token ceiling about 345 to 410s, - # past the thinking class's 300s. Google's generateContent long-decode - # guide says the server closes a non-streaming connection after 10 - # minutes and recommends a 20-minute client read timeout, and its option - # table gives 20 minutes per request for a non-streaming loop - # (aistudio.google.com/docs/long-decode?generate-content=true, read - # 2026-11-14). If the server closes at 10 minutes, no request on this - # row runs past 600s and the timeout's last 300s never apply; 900s also - # covers the table's 20 minutes. - # The harness doubles the 16,384-token budget once, to the ceiling - # below, and records any output still missing there as + # past the thinking class's 300s. The one-question probe ran slower on + # 15 of its 50 calls, down to 53 tokens a second, where a request at + # the ceiling would take about 620s. Google's generateContent + # long-decode guide says the server closes a non-streaming connection + # after 10 minutes and recommends a 20-minute client read timeout, and + # its option table gives 20 minutes per request for a non-streaming + # loop (aistudio.google.com/docs/long-decode?generate-content=true, + # read 2026-11-14). If the server closes at 10 minutes, no request on + # this row runs past 600s, so a request at the ceiling would not finish + # at that slowest rate, and the timeout's last 300s never apply. If + # the table's 20 minutes applies instead, 900s still exceeds the about + # 345 to 410s the gauntlet's rates predict for a request at the + # ceiling, and the 620s the slowest probe rate predicts. The harness + # doubles the 16,384-token budget once, to the ceiling below, and + # records any output still missing there as # budget_exhausted_at_ceiling, so a household that needs more than # 32,768 completion tokens cannot finish on this row. # - provider_max_completion_tokens 32_768: the Models API's @@ -633,7 +644,8 @@ class ModelCard: # which that guide gives for streaming and SDK calls. # - expected_cost_per_scenario_usd 0.075: the gauntlet's whole-scenario # probe, 1,271 prompt and 7,257 completion tokens, $0.075112 at $2/$10 - # per 1M, Google's rate through 2026-11-30. + # per 1M, Google's rate through 2026-11-30. The smoke's two households + # cost $0.100 and $0.081 with no escalation. "gemini/gemini-4-carbon": ModelCard( litellm_id="gemini/gemini-4-carbon", answer_contract="json", @@ -650,7 +662,10 @@ class ModelCard: "sent the board's forced answer tool first, and the Gemini API " "accepted the forced call, but on the 3-variable probe the model " "reasoned until the 16,384-token budget ran out (16,381 reasoning " - "tokens, 0 of 3 outputs parsed, 203s). The gauntlet then passed " + "tokens, 0 of 3 outputs parsed, 203s). The gauntlet sends each " + "probe once, with no budget escalation, so the tool request ran " + "at 16,384 tokens, half the minimum that Google's Gemini 4 Carbon " + "guide recommends for this model. The gauntlet then passed " "on the JSON contract, 3/3 (9,964 completion tokens, 105s) and " "16/16 whole-scenario (7,257 tokens, 79s), about $0.075 per " "scenario at $2/$10 per 1M; the tool contract was not re-tested. " @@ -681,12 +696,15 @@ class ModelCard: "same 32,768 tokens for one direct non-streaming REST call, which " "is what the harness sends; the model page lists 65,536 as the " "default synchronous output, which the guide gives for streaming " - "and SDK calls. The timeout is 900s: Google recommends client " - "timeouts of at least 15 minutes, and a retry at the 32,768-token " - "ceiling takes about 345 to 410s at the gauntlet's 80 to 95 " - "completion tokens a second. Google's generateContent guide says " - "the server closes a non-streaming call after 10 minutes, which " - "would end a request at 600s. The harness doubles the " + "and SDK calls. The card sets a 900s timeout: Google's Gemini 4 Carbon " + "guide recommends client timeouts of at least 15 minutes, and a " + "retry at the 32,768-token ceiling takes about 345 to 410s at the " + "gauntlet's 80 to 95 completion tokens a second. Google's " + "generateContent long-decode guide says the server closes a " + "non-streaming call after 10 minutes, which would end a request " + "at 600s. The same guide recommends a 20-minute client read " + "timeout, and its option table gives 20 minutes per request. The " + "harness doubles the " "16,384-token budget once, to the 32,768-token ceiling, and " "records any output still missing there as " "budget_exhausted_at_ceiling. Google lists $2/$10 per 1M through " @@ -694,12 +712,14 @@ class ModelCard: ), ), # Gemini 4 Hydrogen. Every field below comes from the 2026-11-14 - # onboarding record in ~/reviews/policybench-gemini-4-carbon/{probes,onboard} + # onboarding record in + # ~/reviews/policybench-gemini-4-carbon/{probes,onboard,smoke} # (a raw generateContent probe, a one-question reasoning probe of 10 calls # a shape, the gauntlet report, run with no card from a checkout of - # origin/main, and a harness-path probe of the board's own request, 16 + # origin/main, a harness-path probe of the board's own request, 16 # calls a shape, run from an export of this card's branch before its - # rebase onto main, with the same request code) and from Google's pages: + # rebase onto main, with the same request code, and a 2-household smoke + # run on this card) and from Google's pages: # - answer_contract "tool": the Gemini API accepts a forced function call # on this model (litellm sends mode ANY with allowed_function_names), # and the gauntlet passed on it, 3/3 (10,956 completion tokens, 48s) and @@ -721,14 +741,17 @@ class ModelCard: # - request_timeout_seconds 600. At the gauntlet's 228 to 234 completion # tokens a second, a request at the 65,536-token ceiling (the second # budget escalation) takes about 280 to 290s, and at the slowest rates - # in the record (about 215 to 220 a second, in the one-question probe - # and the smoke) about 300s, the thinking class's timeout. Google's - # generateContent long-decode guide says the server closes a - # non-streaming connection after 10 minutes, though its option table + # in the record (about 215 to 220 a second, in the one-question probe, + # the harness-path probe and the smoke) about 300s, the thinking + # class's timeout. Google's generateContent long-decode guide says the + # server closes a non-streaming connection after 10 minutes and + # recommends a 20-minute client read timeout, and its option table # gives 20 minutes per request # (aistudio.google.com/docs/long-decode?generate-content=true, read # 2026-11-14). If the server closes at 10 minutes, a client timeout - # above 600s could not help this request. + # above 600s could not help this request. If the table's 20 minutes + # applies instead, 600s still exceeds the about 300s the record's + # slowest rates predict for a request at the ceiling. # - provider_max_completion_tokens 65_536: the Models API's # outputTokenLimit for the model, whose entry lists generateContent # among its supported methods (models.get on @@ -743,7 +766,11 @@ class ModelCard: # $1.50/$7.50 per 1M. The gauntlet report gives the cost to three places # and no prompt count; the 2,762 is the prompt count Google returned for # the harness-path probe's forced call on scenario 0, the same request - # (the same messages, answer tool and 16,384-token budget). + # (the same messages, answer tool and 16,384-token budget). The rest of + # the record runs higher: the smoke's two households cost $0.117 and + # $0.109 with no escalation, and 6 of the harness-path probe's 16 + # forced calls ran out of the starting budget, where the board doubles + # the budget and asks again. "gemini/gemini-4-hydrogen": ModelCard( litellm_id="gemini/gemini-4-hydrogen", answer_contract="tool", @@ -795,11 +822,13 @@ class ModelCard: "generateContent long-decode guide gives the same 65,536 tokens " "for every call on this model, streaming or not. The gauntlet " "used 67% and 73% of the 16,384-token starting budget. The " - "timeout is 600s: a request at the 65,536-token ceiling takes " + "card sets a 600s timeout: a request at the 65,536-token ceiling takes " "about 280 to 290s at the gauntlet's 228 to 234 completion tokens " - "a second, close to the thinking class's 300s, and Google's " - "generateContent guide says the server closes a non-streaming " - "call after 10 minutes. Google lists $1.50/$7.50 per 1M." + "a second, close to the thinking class's 300s. Google's " + "generateContent long-decode guide says the server closes a " + "non-streaming call after 10 minutes. The same guide recommends " + "a 20-minute client read timeout, and its option table gives 20 " + "minutes per request. Google lists $1.50/$7.50 per 1M." ), ), "openrouter/z-ai/glm-5.3": ModelCard( From a72462a4eb7a13fd50060517057bc6c8a902ea46 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 15 Nov 2026 09:38:49 -0500 Subject: [PATCH 7/7] Say what Carbon's 32,768-token ceiling bounds and when the harness records exhaustion Carbon's card said a household that needs more than 32,768 completion tokens cannot finish, and that the harness records any output still missing at the ceiling as budget_exhausted_at_ceiling. The ceiling bounds each response: the harness keeps the values a truncated response returned and asks again for the outputs still missing, so a household's requests can total more. It records the exhaustion label only when a length-truncated response at the ceiling leaves outputs missing. The comment and the notes now say so. The cost test's docstring says where Gemini 4 Hydrogen's prompt count comes from. Co-Authored-By: Claude Fable 5.2 --- policybench/model_cards.py | 17 ++++++++++------- tests/test_model_cards.py | 4 +++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index 459cd74c..3a57d0fe 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -626,10 +626,13 @@ class ModelCard: # the table's 20 minutes applies instead, 900s still exceeds the about # 345 to 410s the gauntlet's rates predict for a request at the # ceiling, and the 620s the slowest probe rate predicts. The harness - # doubles the 16,384-token budget once, to the ceiling below, and - # records any output still missing there as - # budget_exhausted_at_ceiling, so a household that needs more than - # 32,768 completion tokens cannot finish on this row. + # can double the request budget once, from 16,384 to the ceiling + # below. If a length-truncated response at that ceiling leaves outputs + # missing, the harness records those outputs as + # budget_exhausted_at_ceiling. The ceiling bounds each response, so a + # household's requests can total more than 32,768 completion tokens: + # the harness keeps the values a truncated response returned and asks + # again for the outputs still missing. # - provider_max_completion_tokens 32_768: the Models API's # outputTokenLimit for the model, whose entry lists generateContent # among its supported methods (models.get on @@ -704,9 +707,9 @@ class ModelCard: "non-streaming call after 10 minutes, which would end a request " "at 600s. The same guide recommends a 20-minute client read " "timeout, and its option table gives 20 minutes per request. The " - "harness doubles the " - "16,384-token budget once, to the 32,768-token ceiling, and " - "records any output still missing there as " + "harness can double the request budget once, from 16,384 to " + "32,768 tokens. If a length-truncated response at that ceiling " + "leaves outputs missing, the harness records those outputs as " "budget_exhausted_at_ceiling. Google lists $2/$10 per 1M through " "2026-11-30 and $4/$20 from 2026-12-01." ), diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index dbbf04b2..cd426023 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -156,7 +156,9 @@ def test_gemini_4_carbon_and_hydrogen_cards_record_the_gauntlet_cost( The token counts are the gauntlet's whole-scenario probe's, and the cost is their reconstruction at the model's price override, so the card, the - counts and the override cannot drift apart. + counts and the override cannot drift apart. Gemini 4 Hydrogen's gauntlet + report gives no prompt count: 2,762 is the count Google returned for the + harness-path probe's forced call on scenario 0, the same request. """ reconstructed = _reconstruct_token_cost( model_name=model_name,