diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts index 3f4d03a1..a605bcb6 100644 --- a/app/src/modelMeta.ts +++ b/app/src/modelMeta.ts @@ -152,6 +152,8 @@ export const MODEL_RELEASE_DATES: Record = { "gemini-3.7-flash": "2026-08-13", "gemini-3.5-flash-lite": "2026-07-21", "gemini-4-argon": "2026-10-13", + "gemini-4-carbon": "2026-11-10", + "gemini-4-hydrogen": "2026-11-12", "gemini-3.1-flash-lite-preview": "2026-03-03", "gpt-5.4-mini": "2026-03-17", "gpt-5.4-nano": "2026-03-17", diff --git a/policybench/config.py b/policybench/config.py index 693c25b1..dee10e7a 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -103,6 +103,22 @@ # the first-party Gemini API like the other Gemini rows. The id is the # model page's one stable version; Google lists no dated snapshot. "gemini-4-argon": "gemini/gemini-4-argon", + # Gemini 4 Carbon. Google opened it to the public on 2026-11-10, when the + # Gemini API changelog lists gemini-4-carbon as generally available + # (aistudio.google.com/docs/changelog, read 2026-11-13); Google's post of + # that day says it had released the model two weeks earlier to + # cybersecurity defenders through its Fairwind Program. It runs on the + # first-party Gemini API like the other Gemini rows. The id is the model + # page's one stable version; Google lists no dated snapshot + # (aistudio.google.com/docs/models/gemini-4-carbon, read 2026-11-13). + "gemini-4-carbon": "gemini/gemini-4-carbon", + # Gemini 4 Hydrogen. The Gemini API changelog lists gemini-4-hydrogen as + # generally available on 2026-11-12, on the free and the paid tier + # (aistudio.google.com/docs/changelog, read 2026-11-13). It runs on the + # first-party Gemini API like the other Gemini rows. The id is the model + # page's one stable version; Google lists no dated snapshot + # (aistudio.google.com/docs/models/gemini-4-hydrogen, read 2026-11-13). + "gemini-4-hydrogen": "gemini/gemini-4-hydrogen", # Cloaked OpenRouter preview listed 2026-08-20 (free window); Z.ai later # identified it as GLM-5.3-Flash. On the board as a labeled preview row. "ox-alpha": "openrouter/stealth/ox-alpha", @@ -245,6 +261,40 @@ # three rates (read 2026-10-13); the override keeps the row priced if the # map is unavailable or changes on 2026-12-01. "gemini-4-argon": {"input": 2.0, "output": 10.0, "cache_read": 0.10}, + # gemini-4-carbon: $2 / $10 per 1M input/output tokens (output includes + # thinking tokens) and $0.10 cached input, the standard-tier rates Google + # lists "through November 30, 2026"; from 2026-12-01 they double to + # $4 / $20 and $0.20 (aistudio.google.com/docs/pricing, read 2026-11-13). + # The board prices a row at the rate in force on its run day, as it + # does gemini-4-argon's. The cost reconstruction takes no date, so this + # entry prices a request at these rates whenever it is made; they are + # Google's rates only for requests made through 2026-11-30. Change this + # entry to 4.0 / 20.0 / 0.20 before any gemini-4-carbon request made on + # or after 2026-12-01. The code prices some rows after Google served + # the response: the harness prices a batch result when it collects it + # (batch_eval.rows_from_unit and _batch_spend_record); it prices a + # response the LiteLLM disk cache replays at the replay + # (eval_no_tools._extract_usage_metadata; eval-no-tools enables the + # cache); and, for a model whose usage records no cost, + # analysis.model_cost_latency computes one from the current entry's + # input and output rates. Once the entry changes, each would put the + # December rate on a request made in November. The pricing page + # lists one rate at every prompt length, implicit caching only, and no + # free tier for this model. litellm 1.95.0's bundled price map has no + # gemini-4 id, and its remote map priced the 2026-11-14 gauntlet's calls + # at the same $2 / $10; the override prices a request from the rates + # above whether or not the map is available. + "gemini-4-carbon": {"input": 2.0, "output": 10.0, "cache_read": 0.10}, + # gemini-4-hydrogen: $1.50 / $7.50 per 1M input/output tokens (output + # includes thinking tokens) and $0.075 cached input, the paid tier's + # standard rates, with no dated change (aistudio.google.com/docs/pricing, + # read 2026-11-13). The page lists one rate at every prompt length and + # implicit caching only; it also lists a free tier, which the row's cost + # does not use. litellm 1.95.0's bundled price map has no gemini-4 id, and + # its remote map priced the 2026-11-14 gauntlet's calls at the same + # $1.50 / $7.50; the override prices a request from the rates above + # whether or not the map is available. + "gemini-4-hydrogen": {"input": 1.5, "output": 7.5, "cache_read": 0.075}, # glm-5.3: $1.40 / $4.40 per 1M (OpenRouter list, 2026-09-03). "glm-5.3": {"input": 1.40, "output": 4.40}, # deepseek dated snapshots via OpenRouter (list prices 2026-09-03). diff --git a/policybench/model_cards.py b/policybench/model_cards.py index 9a10e6be..3a57d0fe 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -570,6 +570,270 @@ class ModelCard: "shows only that the parameter is accepted." ), ), + # Gemini 4 Carbon. Every field below comes from the 2026-11-14 onboarding + # record in ~/reviews/policybench-gemini-4-carbon/{probes,onboard,smoke} + # (a raw generateContent probe, a one-question reasoning probe of 10 calls + # a shape, the gauntlet report, run with no card from a checkout of + # origin/main, and a 2-household smoke run on this card; the gauntlet's + # exact token counts are in onboard/carbon/usage-rows.jsonl) and from + # Google's pages: + # - answer_contract "json". The gauntlet's 3-variable probe sent the + # board's forced answer tool (litellm sends mode ANY with + # allowed_function_names), and the API accepted the forced call, but the + # model reasoned until the 16,384-token budget ran out: 16,384 + # completion tokens, 16,381 of them reasoning, finish=length, 0 of 3 + # outputs parsed, 203s. The gauntlet sends each probe once, with no + # budget escalation, so the tool request ran at half the + # max_output_tokens Google's Gemini 4 Carbon guide recommends: "at + # least 32,768, since Gemini 4 Carbon can spend more of the limit on + # thinking" (aistudio.google.com/docs/whats-new-gemini-4-carbon, read + # 2026-11-13). The gauntlet then passed on the JSON contract, 3/3 + # (9,964 completion tokens, 105s) and 16/16 whole-scenario (7,257 + # tokens, 79s), and suggested "json". The onboarding procedure, written + # before the runs, makes the card JSON when the tool probe fails, and the + # tool contract was not re-tested: re-running the gauntlet until the tool + # probe passes would choose the contract on a retry. So the request + # declares no tool and asks for a JSON object, as the older Gemini rows' + # requests do by the family default. + # - explanation_chunk_size None: 16/16 on the canonical whole-scenario + # request (chunking is closed to new models anyway). + # - thinking_budget True: the model reasons by default. Google's thinking + # guide lists its default as "On (medium)", with levels low, medium and + # high (aistudio.google.com/docs/generate-content/thinking, read + # 2026-11-13), and all 51 raw generateContent responses in the record + # carry thought tokens. The 16,384-token starting budget is not the + # card's: the harness gives every gemini/ id 16,384 + # (GEMINI_JSON_MAX_COMPLETION_TOKENS). The gauntlet used all of it on the + # forced-tool probe, 61% on the 3-variable JSON probe and 44% on the + # whole-scenario JSON probe (22% of the 32,768-token ceiling). + # - request_timeout_seconds 900. Google's Gemini 4 Carbon guide says hard + # prompts "can take several minutes" and recommends HTTP client + # timeouts of "at least 15 minutes" + # (aistudio.google.com/docs/whats-new-gemini-4-carbon, read 2026-11-13). + # The gauntlet's probes ran at about 80 to 95 completion tokens a + # second, so a first request at the 16,384-token budget takes about 170 + # to 205s and a retry at the 32,768-token ceiling about 345 to 410s, + # past the thinking class's 300s. The one-question probe ran slower on + # 15 of its 50 calls, down to 53 tokens a second, where a request at + # the ceiling would take about 620s. Google's generateContent + # long-decode guide says the server closes a non-streaming connection + # after 10 minutes and recommends a 20-minute client read timeout, and + # its option table gives 20 minutes per request for a non-streaming + # loop (aistudio.google.com/docs/long-decode?generate-content=true, + # read 2026-11-14). If the server closes at 10 minutes, no request on + # this row runs past 600s, so a request at the ceiling would not finish + # at that slowest rate, and the timeout's last 300s never apply. If + # the table's 20 minutes applies instead, 900s still exceeds the about + # 345 to 410s the gauntlet's rates predict for a request at the + # ceiling, and the 620s the slowest probe rate predicts. The harness + # can double the request budget once, from 16,384 to the ceiling + # below. If a length-truncated response at that ceiling leaves outputs + # missing, the harness records those outputs as + # budget_exhausted_at_ceiling. The ceiling bounds each response, so a + # household's requests can total more than 32,768 completion tokens: + # the harness keeps the values a truncated response returned and asks + # again for the outputs still missing. + # - provider_max_completion_tokens 32_768: the Models API's + # outputTokenLimit for the model, whose entry lists generateContent + # among its supported methods (models.get on + # generativelanguage.googleapis.com/v1beta/models/gemini-4-carbon, read + # 2026-11-13). Google's generateContent long-decode guide gives the same + # cap for the request the harness sends, one direct non-streaming REST + # generateContent call: such calls on gemini-4-carbon "Default to 32K + # (32,768) output tokens", and a larger max_output_tokens without a + # continuation_token returns a 400 error + # (aistudio.google.com/docs/long-decode?generate-content=true, read + # 2026-11-14). The model page lists 65,536 as "default synchronous", + # which that guide gives for streaming and SDK calls. + # - expected_cost_per_scenario_usd 0.075: the gauntlet's whole-scenario + # probe, 1,271 prompt and 7,257 completion tokens, $0.075112 at $2/$10 + # per 1M, Google's rate through 2026-11-30. The smoke's two households + # cost $0.100 and $0.081 with no escalation. + "gemini/gemini-4-carbon": ModelCard( + litellm_id="gemini/gemini-4-carbon", + answer_contract="json", + request_timeout_seconds=900, + thinking_budget=True, + provider_max_completion_tokens=32_768, + expected_cost_per_scenario_usd=0.075, + notes=( + "Onboarded 2026-11-14. Google opened the model to the public on " + "2026-11-10 (the Gemini API changelog lists gemini-4-carbon as " + "generally available that day; an earlier release went to " + "cybersecurity defenders through its Fairwind Program). The row " + "answers on the JSON contract. The gauntlet, run with no card, " + "sent the board's forced answer tool first, and the Gemini API " + "accepted the forced call, but on the 3-variable probe the model " + "reasoned until the 16,384-token budget ran out (16,381 reasoning " + "tokens, 0 of 3 outputs parsed, 203s). The gauntlet sends each " + "probe once, with no budget escalation, so the tool request ran " + "at 16,384 tokens, half the minimum that Google's Gemini 4 Carbon " + "guide recommends for this model. The gauntlet then passed " + "on the JSON contract, 3/3 (9,964 completion tokens, 105s) and " + "16/16 whole-scenario (7,257 tokens, 79s), about $0.075 per " + "scenario at $2/$10 per 1M; the tool contract was not re-tested. " + "Gemini 4 Argon's rule, a tool_choice auto sensitivity run only " + "if forcing the tool suppresses reasoning, applies only to a row " + "that answers through the forced tool, so this row has no such " + "run. The harness sends no thinkingConfig, and Google's thinking " + "guide lists the model's default as On (medium). Unlike Argon, " + "the model reasoned on every call of a one-question " + "generateContent probe (10 calls a shape): thought tokens on all " + "50 calls, under a forced function call, tool_choice auto, mode " + "ANY with no named function, a JSON response type and no tools " + "(Argon on the same probe: 6 forced, 7 auto, 5 with mode ANY and " + "no named function, and none with a JSON response type or no " + "tools). On that one-question request, unlike the gauntlet's, " + "the forced calls returned a function call 10 times out of 10, " + "with a median of about 2,515 thought tokens against about 2,988 " + "under auto. " + "generateContent reports the count as " + "usageMetadata.thoughtsTokenCount (on all 51 raw responses), " + "litellm 1.95.0 maps it to reasoning_tokens and counts it inside " + "completion_tokens, and the harness records it as each row's " + "reasoning_tokens (16,381, 9,318 and 6,056 on the three gauntlet " + "calls). Every response names the model as gemini-4-carbon, the " + "requested name, with no dated snapshot (modelVersion on the 51 " + "raw responses). The Models API gives an outputTokenLimit of " + "32,768, and Google's generateContent long-decode guide gives the " + "same 32,768 tokens for one direct non-streaming REST call, which " + "is what the harness sends; the model page lists 65,536 as the " + "default synchronous output, which the guide gives for streaming " + "and SDK calls. The card sets a 900s timeout: Google's Gemini 4 Carbon " + "guide recommends client timeouts of at least 15 minutes, and a " + "retry at the 32,768-token ceiling takes about 345 to 410s at the " + "gauntlet's 80 to 95 completion tokens a second. Google's " + "generateContent long-decode guide says the server closes a " + "non-streaming call after 10 minutes, which would end a request " + "at 600s. The same guide recommends a 20-minute client read " + "timeout, and its option table gives 20 minutes per request. The " + "harness can double the request budget once, from 16,384 to " + "32,768 tokens. If a length-truncated response at that ceiling " + "leaves outputs missing, the harness records those outputs as " + "budget_exhausted_at_ceiling. Google lists $2/$10 per 1M through " + "2026-11-30 and $4/$20 from 2026-12-01." + ), + ), + # Gemini 4 Hydrogen. Every field below comes from the 2026-11-14 + # onboarding record in + # ~/reviews/policybench-gemini-4-carbon/{probes,onboard,smoke} + # (a raw generateContent probe, a one-question reasoning probe of 10 calls + # a shape, the gauntlet report, run with no card from a checkout of + # origin/main, a harness-path probe of the board's own request, 16 + # calls a shape, run from an export of this card's branch before its + # rebase onto main, with the same request code, and a 2-household smoke + # run on this card) and from Google's pages: + # - answer_contract "tool": the Gemini API accepts a forced function call + # on this model (litellm sends mode ANY with allowed_function_names), + # and the gauntlet passed on it, 3/3 (10,956 completion tokens, 48s) and + # 16/16 whole-scenario (11,947 tokens, 51s), finish=tool_calls, so the + # row takes the board's forced answer tool, as every Gemini card from + # Gemini 3.6 Flash on does except Gemini 4 Carbon's. + # - explanation_chunk_size None: 16/16 on the canonical whole-scenario + # request (chunking is closed to new models anyway). + # - thinking_budget True: the model reasons by default. Google's thinking + # guide lists its default as "On (high)", with levels low, medium, high + # and xhigh (aistudio.google.com/docs/generate-content/thinking, read + # 2026-11-13), and all 51 raw generateContent responses in the record + # carry thought tokens. The harness sends no thinkingConfig, so this + # model's default is high where Carbon's and Argon's is medium. The + # 16,384-token starting budget is not the card's: the harness gives every + # gemini/ id 16,384 (GEMINI_JSON_MAX_COMPLETION_TOKENS). The gauntlet + # used 67% of it on the 3-variable probe and 73% on the whole-scenario + # probe (18% of the 65,536-token ceiling). + # - request_timeout_seconds 600. At the gauntlet's 228 to 234 completion + # tokens a second, a request at the 65,536-token ceiling (the second + # budget escalation) takes about 280 to 290s, and at the slowest rates + # in the record (about 215 to 220 a second, in the one-question probe, + # the harness-path probe and the smoke) about 300s, the thinking + # class's timeout. Google's generateContent long-decode guide says the + # server closes a non-streaming connection after 10 minutes and + # recommends a 20-minute client read timeout, and its option table + # gives 20 minutes per request + # (aistudio.google.com/docs/long-decode?generate-content=true, read + # 2026-11-14). If the server closes at 10 minutes, a client timeout + # above 600s could not help this request. If the table's 20 minutes + # applies instead, 600s still exceeds the about 300s the record's + # slowest rates predict for a request at the ceiling. + # - provider_max_completion_tokens 65_536: the Models API's + # outputTokenLimit for the model, whose entry lists generateContent + # among its supported methods (models.get on + # generativelanguage.googleapis.com/v1beta/models/gemini-4-hydrogen, read + # 2026-11-13). Google's generateContent long-decode guide gives the same + # cap for every call on this model: "all gemini-4-hydrogen calls" + # default to 64K (65,536) output tokens, streaming and non-streaming + # (aistudio.google.com/docs/long-decode?generate-content=true, read + # 2026-11-14). + # - expected_cost_per_scenario_usd 0.094: the gauntlet's whole-scenario + # probe, 2,762 prompt and 11,947 completion tokens, $0.0937455 at + # $1.50/$7.50 per 1M. The gauntlet report gives the cost to three places + # and no prompt count; the 2,762 is the prompt count Google returned for + # the harness-path probe's forced call on scenario 0, the same request + # (the same messages, answer tool and 16,384-token budget). The rest of + # the record runs higher: the smoke's two households cost $0.117 and + # $0.109 with no escalation, and 6 of the harness-path probe's 16 + # forced calls ran out of the starting budget, where the board doubles + # the budget and asks again. + "gemini/gemini-4-hydrogen": ModelCard( + litellm_id="gemini/gemini-4-hydrogen", + answer_contract="tool", + request_timeout_seconds=600, + thinking_budget=True, + provider_max_completion_tokens=65_536, + expected_cost_per_scenario_usd=0.094, + notes=( + "Onboarded 2026-11-14. The Gemini API changelog lists " + "gemini-4-hydrogen as generally available on 2026-11-12, on the " + "free and the paid tier. The Gemini API accepts a forced function " + "call on this model, so the row takes the board's forced answer " + "tool: the gauntlet, run with no card, passed 3/3 (10,956 " + "completion tokens, 48s) and 16/16 whole-scenario (11,947 tokens, " + "51s) on the tool contract, about $0.094 per scenario at " + "$1.50/$7.50 per 1M. The harness sends no thinkingConfig, and " + "Google's thinking guide lists the model's default as On (high), " + "where Gemini 4 Carbon's and Argon's is medium. Unlike Argon, the " + "model reasoned on every call of a one-question generateContent " + "probe (10 calls a shape): thought tokens on all 50 calls, under " + "a forced function call, tool_choice auto, mode ANY with no named " + "function, a JSON response type and no tools, with a median of " + "5,273 thought tokens forced against about 5,496 under auto, and the " + "forced calls returned a function call 10 times out of 10. " + "On the board's own request for the frozen manifest's first 16 " + "scenarios (16 calls a shape, under a rule fixed before the first " + "call), the forced tool returned reasoning tokens on 16 of 16 " + "calls, tool_choice auto on 16 of 16 and the JSON contract on 16 " + "of 16 (forced against auto, two-sided Fisher exact p = 1.0), so " + "forcing the tool did not lower how often the model reasoned, and " + "the row has no tool_choice auto sensitivity run (Gemini 4 Argon's " + "rule adds one only if forcing the tool suppresses reasoning). " + "That probe sent each request once, with no budget escalation: 6 " + "forced, 6 auto and 2 JSON calls ran out of the 16,384-token " + "budget (finish=length, 14,869 to 16,381 reasoning tokens each), " + "the forced and auto calls on the same six scenarios, so 184 of " + "334 outputs parsed under each tool mode and 262 under JSON; on " + "such a call " + "the board doubles the budget, up to the 65,536-token ceiling, " + "and asks again. " + "generateContent reports the thought-token count as " + "usageMetadata.thoughtsTokenCount (on all 51 raw responses), and " + "litellm 1.95.0 maps it to reasoning_tokens and counts it inside " + "completion_tokens (checked on the saved responses), which the " + "harness records as each row's reasoning_tokens. Every response " + "names the model as gemini-4-hydrogen, the requested name, with " + "no dated snapshot (modelVersion on the 51 raw responses). The " + "Models API gives an outputTokenLimit of 65,536, and Google's " + "generateContent long-decode guide gives the same 65,536 tokens " + "for every call on this model, streaming or not. The gauntlet " + "used 67% and 73% of the 16,384-token starting budget. The " + "card sets a 600s timeout: a request at the 65,536-token ceiling takes " + "about 280 to 290s at the gauntlet's 228 to 234 completion tokens " + "a second, close to the thinking class's 300s. Google's " + "generateContent long-decode guide says the server closes a " + "non-streaming call after 10 minutes. The same guide recommends " + "a 20-minute client read timeout, and its option table gives 20 " + "minutes per request. Google lists $1.50/$7.50 per 1M." + ), + ), "openrouter/z-ai/glm-5.3": ModelCard( litellm_id="openrouter/z-ai/glm-5.3", answer_contract="tool", diff --git a/policybench/paper_results.py b/policybench/paper_results.py index b0fa50c4..9297b3fc 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -3621,6 +3621,22 @@ def last_engine_upgrade(self) -> EngineUpgrade: # Program cyber partners only (techcrunch.com 2026-09-30); that day is # excluded under the public-availability rule. "gemini-4-argon": "2026-10-13", + # aistudio.google.com/docs/changelog ("November 10, 2026": Gemini 4 + # Carbon, gemini-4-carbon, generally available) and blog.google/ + # innovation-and-ai/models-and-research/gemini-models/ + # gemini-4-carbon-now-available (2026-11-10: the API in Google AI Studio + # "for all paying developers and enterprises", and Google AI Ultra + # subscribers in the Gemini app), both read 2026-11-13. The post says + # Google had released the model two weeks earlier to cybersecurity + # defenders through its Fairwind Program; that partner-only period is + # excluded under the public-availability rule. + "gemini-4-carbon": "2026-11-10", + # aistudio.google.com/docs/changelog ("November 12, 2026": Gemini 4 + # Hydrogen, gemini-4-hydrogen, generally available, on the free tier and + # the paid tier) and blog.google/innovation-and-ai/models-and-research/ + # gemini-models/introducing-gemini-4-hydrogen (2026-11-12: "Starting + # today" in the Gemini API and the Gemini app), both read 2026-11-13. + "gemini-4-hydrogen": "2026-11-12", # blog.google gemini-3-1-flash-lite; siliconangle.com 2026-03-03 "gemini-3.1-flash-lite-preview": "2026-03-03", # openai.com/index/introducing-gpt-5-4-mini-and-nano; 9to5mac 2026-03-17 diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index ce00440d..7dfac707 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -3281,6 +3281,143 @@ def cost(cached_prompt_tokens): assert cost(prompt_tokens) - 1e-12 <= cost(cached) <= cost(0) + 1e-12 +def test_gemini_4_carbon_runs_first_party_at_googles_dated_rate(): + """Gemini 4 Carbon runs on the first-party Gemini API, like the other + Gemini rows, at the rate Google's pricing page lists through 2026-11-30: + $2 / $10 per 1M input/output tokens and $0.10 cached input, at every + prompt length (the page doubles all three from 2026-12-01). The override + holds that one rate and the reconstruction takes no date, so the entry + prices a request at these rates whenever it is made.""" + assert MODELS["gemini-4-carbon"] == "gemini/gemini-4-carbon" + assert PRICE_OVERRIDES_PER_1M["gemini-4-carbon"] == { + "input": 2.0, + "output": 10.0, + "cache_read": 0.10, + } + + +@pytest.mark.parametrize( + ("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"), + [ + (1_000_000, 0, 0, 2.0), + (0, 1_000_000, 0, 10.0), + (1_000_000, 1_000_000, 0, 12.0), + (1_000_000, 0, 1_000_000, 0.10), + (1_000_000, 0, 250_000, 1.525), + # The card-free gauntlet's whole-scenario probe (json-full): 1,271 + # prompt and 7,257 completion tokens (6,056 of them reasoning), no + # cached input, the counts the harness's usage extraction returned + # for that call. The gauntlet priced it at $0.075112 from litellm's + # map at the same rates; that is the card's $0.075. + (1_271, 7_257, 0, 0.075112), + ], +) +def test_gemini_4_carbon_override_reconstructs_token_costs( + prompt_tokens, completion_tokens, cached_prompt_tokens, expected +): + reconstructed = _reconstruct_token_cost( + model_name="gemini-4-carbon", + model_id="gemini/gemini-4-carbon", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ) + + assert reconstructed.usd == pytest.approx(expected) + assert reconstructed.is_estimated is False + + +def test_gemini_4_hydrogen_runs_first_party_at_googles_list_rate(): + """Gemini 4 Hydrogen runs on the first-party Gemini API, like the other + Gemini rows, at the paid-tier rate on Google's pricing page: $1.50 / + $7.50 per 1M input/output tokens and $0.075 cached input, at every prompt + length, with no dated change.""" + assert MODELS["gemini-4-hydrogen"] == "gemini/gemini-4-hydrogen" + assert PRICE_OVERRIDES_PER_1M["gemini-4-hydrogen"] == { + "input": 1.5, + "output": 7.5, + "cache_read": 0.075, + } + + +@pytest.mark.parametrize( + ("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"), + [ + (1_000_000, 0, 0, 1.5), + (0, 1_000_000, 0, 7.5), + (1_000_000, 1_000_000, 0, 9.0), + (1_000_000, 0, 1_000_000, 0.075), + (1_000_000, 0, 250_000, 1.14375), + # The card-free gauntlet's whole-scenario probe (tool-full): 2,762 + # prompt and 11,947 completion tokens, priced with no cached input. + # The gauntlet's report gives no prompt count; 2,762 is the count + # Google returned for the harness-path probe's forced call on + # scenario 0, the same request (the same messages, answer tool and + # 16,384-token budget), which reported no cached input. That is the + # card's $0.094. + (2_762, 11_947, 0, 0.0937455), + ], +) +def test_gemini_4_hydrogen_override_reconstructs_token_costs( + prompt_tokens, completion_tokens, cached_prompt_tokens, expected +): + reconstructed = _reconstruct_token_cost( + model_name="gemini-4-hydrogen", + model_id="gemini/gemini-4-hydrogen", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ) + + assert reconstructed.usd == pytest.approx(expected) + assert reconstructed.is_estimated is False + + +# Google's listed per-1M rates (input, cached input, output; Carbon's are the +# rates the page lists through 2026-11-30), written out here and not read from +# PRICE_OVERRIDES_PER_1M, so the property below checks the override against +# the page's figures. +GEMINI_4_LISTED_RATES_PER_1M = { + "gemini-4-carbon": (2.0, 0.10, 10.0), + "gemini-4-hydrogen": (1.5, 0.075, 7.5), +} + + +@given( + model=st.sampled_from(sorted(GEMINI_4_LISTED_RATES_PER_1M)), + prompt_tokens=st.integers(min_value=0, max_value=1_048_576), + completion_tokens=st.integers(min_value=0, max_value=131_072), + cached_share=st.floats(min_value=0.0, max_value=1.0), +) +def test_gemini_4_carbon_and_hydrogen_costs_are_linear_and_bounded_by_the_cache_rate( + model, prompt_tokens, completion_tokens, cached_share +): + """Invariants of each override's reconstruction: the cost is the sum of + uncached input, cached input and output at Google's listed rates; it is + never negative; and it lies between the all-cached and the uncached cost + of the same call.""" + input_rate, cached_rate, output_rate = GEMINI_4_LISTED_RATES_PER_1M[model] + cached = int(prompt_tokens * cached_share) + + def cost(cached_prompt_tokens): + return _reconstruct_token_cost( + model_name=model, + model_id=MODELS[model], + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ).usd + + expected = ( + (prompt_tokens - cached) * input_rate + + cached * cached_rate + + completion_tokens * output_rate + ) / 1_000_000 + assert cost(cached) == pytest.approx(expected, abs=1e-12) + assert cost(cached) >= 0 + assert cost(prompt_tokens) - 1e-12 <= cost(cached) <= cost(0) + 1e-12 + + @pytest.mark.parametrize( ( "prompt_tokens", @@ -3476,6 +3613,76 @@ def test_gemini_4_argon_request_forces_the_answer_tool_and_sets_no_reasoning( assert reasoning_key not in kwargs +def test_gemini_4_carbon_request_asks_for_json_and_sets_no_reasoning(mini_scenario): + """Gemini 4 Carbon's card selects the JSON contract: in the card-free + gauntlet the API accepted the forced answer tool, but the model reasoned + until the 16,384-token budget ran out (0 of 3 outputs parsed), and the + JSON probes passed 3/3 and 16/16. So the request declares no tool and + asks for a JSON object. It carries no reasoning parameter, so the model + reasons at Google's default, medium per Google's thinking guide. It takes + the card's 900s timeout and the Gemini family's 16,384-token starting + budget.""" + from policybench.eval_no_tools import ( + _answer_contract_for_model, + _chat_completion_request_kwargs, + ) + + model_id = "gemini/gemini-4-carbon" + assert _answer_contract_for_model(model_id, env={}) == "json" + assert _thinking_configuration(model_id) == {"mode": "provider_default"} + assert _request_timeout_seconds(model_id) == 900 + + messages, kwargs = _chat_completion_request_kwargs( + mini_scenario, ["income_tax"], model_id + ) + + assert kwargs["response_format"] == {"type": "json_object"} + assert "tools" not in kwargs + assert "tool_choice" not in kwargs + assert "JSON object" in messages[0]["content"] + assert kwargs["max_completion_tokens"] == 16_384 + assert kwargs["timeout"] == 900 + for reasoning_key in ("reasoning", "reasoning_effort", "thinking"): + assert reasoning_key not in kwargs + + +def test_gemini_4_hydrogen_request_forces_the_answer_tool_and_sets_no_reasoning( + mini_scenario, +): + """The Gemini API accepts a forced function call on Gemini 4 Hydrogen and + the card-free gauntlet passed 3/3 and 16/16 on it, so its card selects + the board's forced answer tool, as Gemini 4 Argon's does (the older + Gemini rows answer as JSON by the family default). The request carries no + reasoning parameter, so the model reasons at Google's default, high per + Google's thinking guide. It takes the card's 600s timeout and the Gemini + family's 16,384-token starting budget.""" + from policybench.eval_no_tools import ( + ANSWER_FUNCTION_NAME, + _answer_contract_for_model, + _chat_completion_request_kwargs, + ) + + model_id = "gemini/gemini-4-hydrogen" + assert _answer_contract_for_model(model_id, env={}) == "tool" + assert _thinking_configuration(model_id) == {"mode": "provider_default"} + assert _request_timeout_seconds(model_id) == 600 + + _, kwargs = _chat_completion_request_kwargs(mini_scenario, ["income_tax"], model_id) + + assert kwargs["tool_choice"] == { + "type": "function", + "function": {"name": ANSWER_FUNCTION_NAME}, + } + assert [tool["function"]["name"] for tool in kwargs["tools"]] == [ + ANSWER_FUNCTION_NAME + ] + assert kwargs["max_completion_tokens"] == 16_384 + assert kwargs["timeout"] == 600 + assert "response_format" not in kwargs + for reasoning_key in ("reasoning", "reasoning_effort", "thinking"): + assert reasoning_key not in kwargs + + def test_claude_haiku_5_5_reasons_by_default_but_not_under_the_forced_tool(): """Haiku 5.5 thinks when the request omits the thinking parameter, so it is a thinking-default Claude: provider-default thinking, the thinking diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index a1cf6e4a..cd426023 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -13,6 +13,7 @@ _answer_contract_for_model, _completion_budget_ceiling, _completion_controls, + _reconstruct_token_cost, _request_timeout_seconds, _required_explanation_chunk_size, ) @@ -78,6 +79,8 @@ "openrouter/z-ai/glm-5.3": ("tool", None, 2400, 16_384), "openrouter/meta/muse-wonder": ("json", None, 300, 16_384), "gemini/gemini-4-argon": ("tool", None, 300, 16_384), + "gemini/gemini-4-carbon": ("json", None, 900, 16_384), + "gemini/gemini-4-hydrogen": ("tool", None, 600, 16_384), } @@ -138,6 +141,36 @@ def test_gpt_56_cards_record_measured_full_run_cost(model_id, expected_cost): assert card_for(model_id).expected_cost_per_scenario_usd == expected_cost +@pytest.mark.parametrize( + ("model_name", "model_id", "prompt_tokens", "completion_tokens", "expected_cost"), + [ + ("gemini-4-carbon", "gemini/gemini-4-carbon", 1_271, 7_257, 0.075), + ("gemini-4-hydrogen", "gemini/gemini-4-hydrogen", 2_762, 11_947, 0.094), + ], +) +def test_gemini_4_carbon_and_hydrogen_cards_record_the_gauntlet_cost( + model_name, model_id, prompt_tokens, completion_tokens, expected_cost +): + """Each card carries its card-free gauntlet's whole-scenario cost, rounded + to three places as ``policybench onboard`` suggests it. + + The token counts are the gauntlet's whole-scenario probe's, and the cost is + their reconstruction at the model's price override, so the card, the + counts and the override cannot drift apart. Gemini 4 Hydrogen's gauntlet + report gives no prompt count: 2,762 is the count Google returned for the + harness-path probe's forced call on scenario 0, the same request. + """ + reconstructed = _reconstruct_token_cost( + model_name=model_name, + model_id=model_id, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + ) + assert reconstructed.is_estimated is False + assert round(reconstructed.usd, 3) == expected_cost + assert card_for(model_id).expected_cost_per_scenario_usd == expected_cost + + @pytest.mark.parametrize( ("provider_max", "expected"), [(30_000, 30_000), (200_000, 128_000)] ) @@ -176,6 +209,8 @@ def test_provider_max_clamps_initial_completion_budget(monkeypatch): ("gemini/gemini-3-flash-preview", 65_535), ("gemini/gemini-3.6-flash", 65_536), ("gemini/gemini-4-argon", 32_768), + ("gemini/gemini-4-carbon", 32_768), + ("gemini/gemini-4-hydrogen", 65_536), ], ) def test_roster_model_provider_maximums_limit_escalation(model_id, expected): diff --git a/tests/test_paper_results.py b/tests/test_paper_results.py index a0f154d9..57ef7a1f 100644 --- a/tests/test_paper_results.py +++ b/tests/test_paper_results.py @@ -111,6 +111,23 @@ def test_gemini_4_argon_release_date_is_its_public_day(): assert MODEL_RELEASE_DATES["gemini-4-argon"] == "2026-10-13" +def test_gemini_4_carbon_release_date_is_its_public_day(): + """The Gemini API changelog lists Gemini 4 Carbon as generally available + on 2026-11-10, and Google's post of that day opens the API to all paying + developers and enterprises. The post says Google had released the model + two weeks earlier to cybersecurity defenders through its Fairwind + Program; that partner-only period is excluded under the + public-availability rule, as Gemini 4 Argon's is.""" + assert MODEL_RELEASE_DATES["gemini-4-carbon"] == "2026-11-10" + + +def test_gemini_4_hydrogen_release_date_is_recorded(): + """The Gemini API changelog lists Gemini 4 Hydrogen as generally + available on 2026-11-12, on the free and the paid tier, and Google's + launch post is dated the same day.""" + assert MODEL_RELEASE_DATES["gemini-4-hydrogen"] == "2026-11-12" + + def test_gpt_6_1_terra_release_date_is_its_announcement_day(): """OpenAI announced GPT-6.1 Terra on 2026-10-26 and opened it in ChatGPT (Plus, Pro, Business, Enterprise and Edu), Codex and the API (id