Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/cc-session/benchmark.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func runBenchmark(args []string, out io.Writer, errOut io.Writer, store parser.S
days := fs.Int("days", 30, "how far back to scan")
minKB := fs.Int("min-kb", 100, "minimum JSONL file size in KB")
maxN := fs.Int("n", 10, "max sessions to include")
model := fs.String("model", "opus", "model: opus, opus-4-6, opus-4-7, opus-4-8, sonnet, or fable (fable-5-1)")
model := fs.String("model", "opus", "model: opus, opus-4-6, opus-4-7, opus-4-8, opus-5-5, sonnet, or fable (fable-5-1)")
overhead := fs.Int("overhead", 0, "session overhead tokens (system+tools+CLAUDE.md); measure with a 1-turn session")
isNoAPI := fs.Bool("no-api", false, "skip API calls; estimate filtered-text tokens with chars/2 (offline fallback)")
if err := fs.Parse(reorderArgs(args)); err != nil {
Expand Down
30 changes: 16 additions & 14 deletions docs/benchmark.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ All other parameters are derived automatically from your real session data.
| `--days` | 30 | How far back to scan for sessions |
| `--min-kb` | 100 | Minimum JSONL file size in KB |
| `--n` | 10 | Max successful session results to report |
| `--model` | opus | Pricing and token-counting model: `opus`, `opus-4-6`, `opus-4-7`, `opus-4-8`, `sonnet`, or `fable` (`fable-5-1`) |
| `--model` | opus | Pricing and token-counting model: `opus`, `opus-4-6`, `opus-4-7`, `opus-4-8`, `opus-5-5`, `sonnet`, or `fable` (`fable-5-1`) |

### Example output

Expand All @@ -62,12 +62,12 @@ Median break-even: turn 1 | 10-turn saving: 69% | 100-turn saving: 48%

From [Anthropic prompt caching docs](https://platform.claude.com/docs/en/build-with-claude/prompt-caching):

| Bucket | API field | Rate (Opus) |
|--------|-----------|:-----------:|
| Cache read | `cache_read_input_tokens` | $0.50/M (0.1× base) |
| Cache write | `cache_creation_input_tokens` | $6.25/M (1.25× base) |
| Uncached input | `input_tokens` | $5.00/M (1× base) |
| Output | `output_tokens` | $25/M (excluded — same for both scenarios) |
| Bucket | API field | Rate (Opus 4.x) | Rate (Opus 5.5) |
|--------|-----------|:-----------:|:-----------:|
| Cache read | `cache_read_input_tokens` | $0.50/M (0.1× base) | $0.20/M (0.05× base) |
| Cache write | `cache_creation_input_tokens` | $6.25/M (1.25× base) | $5.00/M (1.25× base) |
| Uncached input | `input_tokens` | $5.00/M (1× base) | $4.00/M (1× base) |
| Output | `output_tokens` | $25/M (excluded — same for both scenarios) | $20/M (excluded) |

### Per-API-call billing

Expand Down Expand Up @@ -162,16 +162,18 @@ the historical one-shot `NewCtx × CacheWrite` behavior.

`X` comes from transcript API usage and `C` comes from the Anthropic token
counting API. The `--model` flag controls both pricing and the tokenizer used by
the token counting API. `opus` is an alias for `opus-4-8`; explicit Opus versions
map to `claude-opus-4-6`, `claude-opus-4-7`, or `claude-opus-4-8`; `sonnet` maps
to `claude-sonnet-4-6`; `fable` and `fable-5-1` both map to `claude-fable-5-1`.
Opus 4.6, 4.7, and 4.8 use the same Opus pricing rates.
the token counting API. `opus` is an alias for `opus-4-8`; explicit Opus 4.x
versions map to `claude-opus-4-6`, `claude-opus-4-7`, or `claude-opus-4-8`;
`opus-5-5` maps to `claude-opus-5-5`; `sonnet` maps to `claude-sonnet-4-6`;
`fable` and `fable-5-1` both map to `claude-fable-5-1`.
Opus 4.6, 4.7, and 4.8 use the same Opus pricing rates; Opus 5.5 has its own,
lower rates (see the pricing table above).
Fallback constants are used only for behavior that cannot be read directly from
transcript usage, such as sparse tool I/O data.

Claude Fable 5.1's cache read is 2.5% of base input, not the 10% every other
model here uses, so its cost-savings numbers are not directly comparable to the
Opus/Sonnet rows.
Cache read as a percentage of base input differs by model: Opus 4.x and Sonnet
use 10%, Opus 5.5 uses 5%, and Claude Fable 5.1 uses 2.5%. Cost-savings numbers
are only directly comparable across rows using the same ratio.

### Simplifications

Expand Down
20 changes: 15 additions & 5 deletions internal/benchmark/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,24 @@ type Pricing struct {
var PricingOpus = Pricing{CachedRead: 0.50, CacheWrite: 6.25, BaseInput: 5.00}
var PricingSonnet = Pricing{CachedRead: 0.30, CacheWrite: 3.75, BaseInput: 3.00}

// PricingFable is Claude Fable 5.1's pricing. Unlike every other tier here, its
// cache read is 2.5% of base input rather than the usual 10% (source: Anthropic
// pricing docs, "a cache hit costs 2.5% of the standard input price"), so its
// cost-savings numbers are not directly comparable to the Opus/Sonnet rows.
// PricingOpus55 is Claude Opus 5.5's pricing. Its cache read is 5% of base
// input rather than the 10% Opus 4.x/Sonnet use (source: Anthropic pricing
// docs, "a cache hit costs 5% of the standard input price"), so its
// cost-savings numbers are not directly comparable to the Opus 4.x/Sonnet rows.
var PricingOpus55 = Pricing{CachedRead: 0.20, CacheWrite: 5.00, BaseInput: 4.00}

// PricingFable is Claude Fable 5.1's pricing. Its cache read is 2.5% of base
// input — lower than the 10% Opus 4.x/Sonnet use and the 5% Opus 5.5 uses
// (source: Anthropic pricing docs, "a cache hit costs 2.5% of the standard
// input price"), so its cost-savings numbers are not directly comparable to
// the other rows.
var PricingFable = Pricing{CachedRead: 0.25, CacheWrite: 12.50, BaseInput: 10.00}

const (
TokenCountModelOpus46 = "claude-opus-4-6"
TokenCountModelOpus47 = "claude-opus-4-7"
TokenCountModelOpus48 = "claude-opus-4-8"
TokenCountModelOpus55 = "claude-opus-5-5"
TokenCountModelSonnet = "claude-sonnet-4-6"
TokenCountModelFable = "claude-fable-5-1"
)
Expand All @@ -43,10 +51,12 @@ func ResolveModel(model string) (ModelConfig, error) {
return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus47}, nil
case "opus-4-6":
return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus46}, nil
case "opus-5-5":
return ModelConfig{Pricing: PricingOpus55, TokenCountModel: TokenCountModelOpus55}, nil
case "fable", "fable-5-1":
return ModelConfig{Pricing: PricingFable, TokenCountModel: TokenCountModelFable}, nil
default:
return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, sonnet, fable, or fable-5-1", model)
return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, opus-5-5, sonnet, fable, or fable-5-1", model)
}
}

Expand Down
28 changes: 27 additions & 1 deletion internal/benchmark/model_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ func Test_ResolveModel_GivenKnownAlias_ThenReturnsExpectedConfig(t *testing.T) {
wantPricing: PricingOpus,
wantTokenCount: TokenCountModelOpus46,
},
{
name: "opus-5-5",
alias: "opus-5-5",
wantPricing: PricingOpus55,
wantTokenCount: TokenCountModelOpus55,
},
{
name: "fable",
alias: "fable",
Expand Down Expand Up @@ -73,7 +79,7 @@ func Test_ResolveModel_GivenKnownAlias_ThenReturnsExpectedConfig(t *testing.T) {
}

// Regression guard: PricingFable's cache read must stay at 2.5% of base input
// ($0.25/MTok), not the 10% ratio every other tier in this file uses.
// ($0.25/MTok), not the 10% ratio Opus 4.x/Sonnet use.
func Test_PricingFable_ThenCacheReadIsTwoPointFivePercentOfBaseInput(t *testing.T) {
want := PricingFable.BaseInput * 0.025
if PricingFable.CachedRead != want {
Expand All @@ -82,6 +88,16 @@ func Test_PricingFable_ThenCacheReadIsTwoPointFivePercentOfBaseInput(t *testing.
}
}

// Regression guard: PricingOpus55's cache read must stay at 5% of base input
// ($0.20/MTok), not the 10% ratio Opus 4.x/Sonnet use.
func Test_PricingOpus55_ThenCacheReadIsFivePercentOfBaseInput(t *testing.T) {
want := PricingOpus55.BaseInput * 0.05
if PricingOpus55.CachedRead != want {
t.Errorf("PricingOpus55.CachedRead = %v, want %v (5%% of BaseInput %v)",
PricingOpus55.CachedRead, want, PricingOpus55.BaseInput)
}
}

func Test_ResolveModel_GivenUnknownAlias_ThenErrorListsFableAliases(t *testing.T) {
_, err := ResolveModel("nonsense")
if err == nil {
Expand All @@ -91,3 +107,13 @@ func Test_ResolveModel_GivenUnknownAlias_ThenErrorListsFableAliases(t *testing.T
t.Errorf("error = %v, want it to list the fable aliases", err)
}
}

func Test_ResolveModel_GivenUnknownAlias_ThenErrorListsOpus55Alias(t *testing.T) {
_, err := ResolveModel("nonsense")
if err == nil {
t.Fatal("ResolveModel(\"nonsense\") returned nil error, want unknown model error")
}
if !strings.Contains(err.Error(), "opus-5-5") {
t.Errorf("error = %v, want it to list the opus-5-5 alias", err)
}
}
Loading