From f4a8085ca0f023ba50951a5241b1d583ecf471c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maple=EF=BC=81?= Date: Wed, 23 Sep 2026 11:05:40 +0800 Subject: [PATCH] feat: add Claude Opus 5.5 as a benchmark pricing model Opus 5.5 (claude-opus-5-5) prices cache reads at 5% of base input ($4 input / $5 cache write / $0.20 cache read), between Opus 4.x's 10% and Fable's 2.5%. The default `opus` alias stays on opus-4-8 so existing baselines remain comparable. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01FxSSNGDv4kKwZZ3oHdhbef --- cmd/cc-session/benchmark.go | 2 +- docs/benchmark.md | 30 ++++++++++++++++-------------- internal/benchmark/model.go | 20 +++++++++++++++----- internal/benchmark/model_test.go | 28 +++++++++++++++++++++++++++- 4 files changed, 59 insertions(+), 21 deletions(-) diff --git a/cmd/cc-session/benchmark.go b/cmd/cc-session/benchmark.go index 2ded2ea..7875d27 100644 --- a/cmd/cc-session/benchmark.go +++ b/cmd/cc-session/benchmark.go @@ -30,7 +30,7 @@ func runBenchmark(args []string, out io.Writer, errOut io.Writer, store parser.S days := fs.Int("days", 30, "how far back to scan") minKB := fs.Int("min-kb", 100, "minimum JSONL file size in KB") maxN := fs.Int("n", 10, "max sessions to include") - model := fs.String("model", "opus", "model: opus, opus-4-6, opus-4-7, opus-4-8, sonnet, or fable (fable-5-1)") + model := fs.String("model", "opus", "model: opus, opus-4-6, opus-4-7, opus-4-8, opus-5-5, sonnet, or fable (fable-5-1)") overhead := fs.Int("overhead", 0, "session overhead tokens (system+tools+CLAUDE.md); measure with a 1-turn session") isNoAPI := fs.Bool("no-api", false, "skip API calls; estimate filtered-text tokens with chars/2 (offline fallback)") if err := fs.Parse(reorderArgs(args)); err != nil { diff --git a/docs/benchmark.md b/docs/benchmark.md index af5c2a6..3b06af1 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -36,7 +36,7 @@ All other parameters are derived automatically from your real session data. | `--days` | 30 | How far back to scan for sessions | | `--min-kb` | 100 | Minimum JSONL file size in KB | | `--n` | 10 | Max successful session results to report | -| `--model` | opus | Pricing and token-counting model: `opus`, `opus-4-6`, `opus-4-7`, `opus-4-8`, `sonnet`, or `fable` (`fable-5-1`) | +| `--model` | opus | Pricing and token-counting model: `opus`, `opus-4-6`, `opus-4-7`, `opus-4-8`, `opus-5-5`, `sonnet`, or `fable` (`fable-5-1`) | ### Example output @@ -62,12 +62,12 @@ Median break-even: turn 1 | 10-turn saving: 69% | 100-turn saving: 48% From [Anthropic prompt caching docs](https://platform.claude.com/docs/en/build-with-claude/prompt-caching): -| Bucket | API field | Rate (Opus) | -|--------|-----------|:-----------:| -| Cache read | `cache_read_input_tokens` | $0.50/M (0.1× base) | -| Cache write | `cache_creation_input_tokens` | $6.25/M (1.25× base) | -| Uncached input | `input_tokens` | $5.00/M (1× base) | -| Output | `output_tokens` | $25/M (excluded — same for both scenarios) | +| Bucket | API field | Rate (Opus 4.x) | Rate (Opus 5.5) | +|--------|-----------|:-----------:|:-----------:| +| Cache read | `cache_read_input_tokens` | $0.50/M (0.1× base) | $0.20/M (0.05× base) | +| Cache write | `cache_creation_input_tokens` | $6.25/M (1.25× base) | $5.00/M (1.25× base) | +| Uncached input | `input_tokens` | $5.00/M (1× base) | $4.00/M (1× base) | +| Output | `output_tokens` | $25/M (excluded — same for both scenarios) | $20/M (excluded) | ### Per-API-call billing @@ -162,16 +162,18 @@ the historical one-shot `NewCtx × CacheWrite` behavior. `X` comes from transcript API usage and `C` comes from the Anthropic token counting API. The `--model` flag controls both pricing and the tokenizer used by -the token counting API. `opus` is an alias for `opus-4-8`; explicit Opus versions -map to `claude-opus-4-6`, `claude-opus-4-7`, or `claude-opus-4-8`; `sonnet` maps -to `claude-sonnet-4-6`; `fable` and `fable-5-1` both map to `claude-fable-5-1`. -Opus 4.6, 4.7, and 4.8 use the same Opus pricing rates. +the token counting API. `opus` is an alias for `opus-4-8`; explicit Opus 4.x +versions map to `claude-opus-4-6`, `claude-opus-4-7`, or `claude-opus-4-8`; +`opus-5-5` maps to `claude-opus-5-5`; `sonnet` maps to `claude-sonnet-4-6`; +`fable` and `fable-5-1` both map to `claude-fable-5-1`. +Opus 4.6, 4.7, and 4.8 use the same Opus pricing rates; Opus 5.5 has its own, +lower rates (see the pricing table above). Fallback constants are used only for behavior that cannot be read directly from transcript usage, such as sparse tool I/O data. -Claude Fable 5.1's cache read is 2.5% of base input, not the 10% every other -model here uses, so its cost-savings numbers are not directly comparable to the -Opus/Sonnet rows. +Cache read as a percentage of base input differs by model: Opus 4.x and Sonnet +use 10%, Opus 5.5 uses 5%, and Claude Fable 5.1 uses 2.5%. Cost-savings numbers +are only directly comparable across rows using the same ratio. ### Simplifications diff --git a/internal/benchmark/model.go b/internal/benchmark/model.go index 11049a0..50fbbbf 100644 --- a/internal/benchmark/model.go +++ b/internal/benchmark/model.go @@ -12,16 +12,24 @@ type Pricing struct { var PricingOpus = Pricing{CachedRead: 0.50, CacheWrite: 6.25, BaseInput: 5.00} var PricingSonnet = Pricing{CachedRead: 0.30, CacheWrite: 3.75, BaseInput: 3.00} -// PricingFable is Claude Fable 5.1's pricing. Unlike every other tier here, its -// cache read is 2.5% of base input rather than the usual 10% (source: Anthropic -// pricing docs, "a cache hit costs 2.5% of the standard input price"), so its -// cost-savings numbers are not directly comparable to the Opus/Sonnet rows. +// PricingOpus55 is Claude Opus 5.5's pricing. Its cache read is 5% of base +// input rather than the 10% Opus 4.x/Sonnet use (source: Anthropic pricing +// docs, "a cache hit costs 5% of the standard input price"), so its +// cost-savings numbers are not directly comparable to the Opus 4.x/Sonnet rows. +var PricingOpus55 = Pricing{CachedRead: 0.20, CacheWrite: 5.00, BaseInput: 4.00} + +// PricingFable is Claude Fable 5.1's pricing. Its cache read is 2.5% of base +// input — lower than the 10% Opus 4.x/Sonnet use and the 5% Opus 5.5 uses +// (source: Anthropic pricing docs, "a cache hit costs 2.5% of the standard +// input price"), so its cost-savings numbers are not directly comparable to +// the other rows. var PricingFable = Pricing{CachedRead: 0.25, CacheWrite: 12.50, BaseInput: 10.00} const ( TokenCountModelOpus46 = "claude-opus-4-6" TokenCountModelOpus47 = "claude-opus-4-7" TokenCountModelOpus48 = "claude-opus-4-8" + TokenCountModelOpus55 = "claude-opus-5-5" TokenCountModelSonnet = "claude-sonnet-4-6" TokenCountModelFable = "claude-fable-5-1" ) @@ -43,10 +51,12 @@ func ResolveModel(model string) (ModelConfig, error) { return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus47}, nil case "opus-4-6": return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus46}, nil + case "opus-5-5": + return ModelConfig{Pricing: PricingOpus55, TokenCountModel: TokenCountModelOpus55}, nil case "fable", "fable-5-1": return ModelConfig{Pricing: PricingFable, TokenCountModel: TokenCountModelFable}, nil default: - return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, sonnet, fable, or fable-5-1", model) + return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, opus-5-5, sonnet, fable, or fable-5-1", model) } } diff --git a/internal/benchmark/model_test.go b/internal/benchmark/model_test.go index 1157b88..6e68153 100644 --- a/internal/benchmark/model_test.go +++ b/internal/benchmark/model_test.go @@ -42,6 +42,12 @@ func Test_ResolveModel_GivenKnownAlias_ThenReturnsExpectedConfig(t *testing.T) { wantPricing: PricingOpus, wantTokenCount: TokenCountModelOpus46, }, + { + name: "opus-5-5", + alias: "opus-5-5", + wantPricing: PricingOpus55, + wantTokenCount: TokenCountModelOpus55, + }, { name: "fable", alias: "fable", @@ -73,7 +79,7 @@ func Test_ResolveModel_GivenKnownAlias_ThenReturnsExpectedConfig(t *testing.T) { } // Regression guard: PricingFable's cache read must stay at 2.5% of base input -// ($0.25/MTok), not the 10% ratio every other tier in this file uses. +// ($0.25/MTok), not the 10% ratio Opus 4.x/Sonnet use. func Test_PricingFable_ThenCacheReadIsTwoPointFivePercentOfBaseInput(t *testing.T) { want := PricingFable.BaseInput * 0.025 if PricingFable.CachedRead != want { @@ -82,6 +88,16 @@ func Test_PricingFable_ThenCacheReadIsTwoPointFivePercentOfBaseInput(t *testing. } } +// Regression guard: PricingOpus55's cache read must stay at 5% of base input +// ($0.20/MTok), not the 10% ratio Opus 4.x/Sonnet use. +func Test_PricingOpus55_ThenCacheReadIsFivePercentOfBaseInput(t *testing.T) { + want := PricingOpus55.BaseInput * 0.05 + if PricingOpus55.CachedRead != want { + t.Errorf("PricingOpus55.CachedRead = %v, want %v (5%% of BaseInput %v)", + PricingOpus55.CachedRead, want, PricingOpus55.BaseInput) + } +} + func Test_ResolveModel_GivenUnknownAlias_ThenErrorListsFableAliases(t *testing.T) { _, err := ResolveModel("nonsense") if err == nil { @@ -91,3 +107,13 @@ func Test_ResolveModel_GivenUnknownAlias_ThenErrorListsFableAliases(t *testing.T t.Errorf("error = %v, want it to list the fable aliases", err) } } + +func Test_ResolveModel_GivenUnknownAlias_ThenErrorListsOpus55Alias(t *testing.T) { + _, err := ResolveModel("nonsense") + if err == nil { + t.Fatal("ResolveModel(\"nonsense\") returned nil error, want unknown model error") + } + if !strings.Contains(err.Error(), "opus-5-5") { + t.Errorf("error = %v, want it to list the opus-5-5 alias", err) + } +}