feat(cache): optimize provider cache economics

.env.template

@@ -117,6 +117,11 @@
 # Model cache uses the local filesystem by default.
 # Set REDIS_URL to use Redis-backed caching instead.
 
+# Add provider-native prompt-cache keys, breakpoints, and cached-content plans
+# after routing (default: true). Set false as an operational kill switch; empty
+# or invalid values keep the default. Client cache directives still pass through.
+# PROVIDER_PROMPT_CACHE_PLANNER_ENABLED=true
+
 # Redis Configuration
 # REDIS_URL=redis://localhost:6379
 # REDIS_KEY_MODELS=gomodel:models

docs/features/cache.mdx

@@ -90,6 +90,40 @@ The exact cache hashes:
 This means guardrails and workflows affect cache keys when they change the
 resolved workflow or the final body sent through execution.
 
+JSON object member order and insignificant whitespace are canonicalized before
+hashing. Requests that differ only in formatting therefore share one exact
+entry. Concurrent identical misses are also coalesced in-process: one request
+reaches the provider and the waiters replay its response. Failed or otherwise
+non-cacheable responses are never fanned out.
+
+## Provider prompt-cache planning
+
+After routing resolves the concrete provider and model, GoModel adds a cache
+plan when the stable prefix meets that provider's minimum size and the client
+did not already supply a cache directive:
+
+- OpenAI receives a stable `prompt_cache_key`; models supporting explicit
+  caching also receive a breakpoint and explicit cache mode.
+- Anthropic receives top-level automatic ephemeral caching.
+- Amazon Bedrock Converse receives a cache point after the stable prefix.
+- Native Gemini AI Studio creates and reuses a five-minute cached-content
+  object, with concurrent creation coalesced by prefix and stable credential.
+
+The plan is applied post-routing so provider-only fields cannot leak through a
+fallback to another backend. Cache keys include the concrete provider instance
+and model, and caller-supplied cache controls always win.
+
+Provider prompt-cache planning is enabled by default. To disable GoModel's
+automatic planning while continuing to pass through compatible client cache
+directives, set:
+
+```bash
+PROVIDER_PROMPT_CACHE_PLANNER_ENABLED=false
+```
+
+The flag accepts Go boolean values. Missing, empty, or invalid values keep the
+safe default (`true`) and an invalid value emits a startup warning.
+
 ## `user_path` behavior
 
 For the exact cache, `user_path` is not added to the cache key by itself.

docs/pro.mdx

@@ -54,10 +54,16 @@ Measured on a corpus of real agent conversations at default settings:
 - anything, if estimated savings fall below the threshold; the request
   passes through byte-identical
 
-Compression is a pure function of the request body, so the same request
-always compresses the same way and the provider's prompt-cache prefix stays
-stable as a conversation grows. It fails open: if the engine errors, the
-request is forwarded unchanged.
+Compression freezes the exact rewritten history already sent for each detected
+session and replays it byte-for-byte in later requests. New turns can still be
+compressed against that history, including slightly modified file reads whose
+unchanged line runs become reconstructable deltas. Very long conversations roll
+forward through bounded compaction epochs without revisiting older epochs.
+
+The final decision is economic: estimated savings on ordinary and expected
+cached reuses must exceed the value of any provider-cache prefix the rewrite
+would destroy. Frozen history normally makes destroyed cache value zero. The
+engine fails open if it cannot safely establish that invariant.
 
 Covered endpoints: `POST /v1/chat/completions`, `POST /v1/messages`, and
 `POST /v1/responses` — including function and custom tool-call outputs on
@@ -104,12 +110,20 @@ prompt-cache reuse.
 
 ## Main settings
 
+These variables are consumed by the separately distributed GoModel Pro
+compression engine. They are intentionally not parsed by the open-core binary
+when the Pro extension is absent.
+
 | Env var | Default | Meaning |
 |---|---|---|
 | `PRO_COMPRESSION_ENABLED` | `true` | Master switch |
 | `PRO_COMPRESSION_SCOPE` | `full_history` | `new_messages_only` rewrites only after the last assistant turn |
 | `PRO_COMPRESSION_EXCLUDE_MODELS` | — | Model globs to skip |
-| `PRO_COMPRESSION_MIN_SAVINGS_TOKENS` | `64` | Minimum estimated savings to rewrite at all |
+| `PRO_COMPRESSION_MIN_SAVINGS_TOKENS` | `64` | Minimum estimated savings before economic evaluation |
+| `PRO_COMPRESSION_EXPECTED_CACHE_REUSES` | `4` | Future cached reuses included in expected savings |
+| `PRO_COMPRESSION_CACHED_INPUT_RATE` | `0.1` | Cached-input price as a fraction of ordinary input |
+| `PRO_COMPRESSION_INPUT_PRICE_PER_MTOK` | `1` | Fallback price scale for monetary audit estimates |
+| `PRO_COMPRESSION_FROZEN_EPOCH_MESSAGES` | `32` | Messages per immutable compaction epoch |
 | `PRO_COMPRESSION_NORMALIZE` | `true` | Master switch for the normalization pass |
 | `PRO_COMPRESSION_STRIP_LINE_NUMBERS` | `true` | Strip the line-number gutter from file reads |
 | `PRO_COMPRESSION_DROP_REASONING` | `true` | Drop replayed reasoning from assistant history |

docs/providers/gemini.mdx

@@ -35,6 +35,12 @@ providers:
   when per-provider `GEMINI_API_MODE` is unset. Prefer `GEMINI_API_MODE`.
 </Note>
 
+Native AI Studio mode can also materialize GoModel's post-routing prompt-cache
+plans as five-minute Gemini cached-content objects. Reuse is scoped to the API
+key selected for the session; sessionless traffic with rotating keys skips
+object creation so a resource is never reused under the wrong credential.
+Vertex currently receives the original request without cached-content objects.
+
 ## Base URLs
 
 GoModel keeps separate internal bases for native Gemini and the

internal/core/types.go

@@ -41,6 +41,18 @@ type ChatRequest struct {
 	User              string            `json:"user,omitempty"`
 	ServiceTier       string            `json:"service_tier,omitempty"`
 	ExtraFields       UnknownJSONFields `json:"-" swaggerignore:"true"`
+	// PromptCachePlan carries gateway-internal, post-routing cache metadata.
+	// It is never serialized to clients or upstream OpenAI-compatible APIs.
+	PromptCachePlan *PromptCachePlan `json:"-" swaggerignore:"true"`
+}
+
+// GatewayCachePointField is the internal marker shared by provider cache
+// planners and native request translators. It is never forwarded verbatim.
+const GatewayCachePointField = "_gomodel_cache_point"
+
+// PromptCachePlan identifies a provider-native cached prefix materialization.
+type PromptCachePlan struct {
+	Key string
 }
 
 func (r *ChatRequest) semanticSelector() (string, string) {

internal/providers/bedrock/bedrock_test.go

@@ -798,3 +798,57 @@ func TestStreamConverter_FormatChunkUsage(t *testing.T) {
 		t.Errorf("total_tokens = %v", usage["total_tokens"])
 	}
 }
+
+func TestGatewayCachePointUsesJSONBooleanAndNeverCreatesEmptyMessage(t *testing.T) {
+	fields := core.UnknownJSONFieldsFromMap(map[string]json.RawMessage{
+		core.GatewayCachePointField: json.RawMessage("  true\n"),
+	})
+	if !isGatewayCachePoint(fields) {
+		t.Fatal("formatted JSON true was not recognized")
+	}
+	falseFields := core.UnknownJSONFieldsFromMap(map[string]json.RawMessage{
+		core.GatewayCachePointField: json.RawMessage(`false`),
+	})
+	if isGatewayCachePoint(falseFields) {
+		t.Fatal("JSON false was recognized as a cache point")
+	}
+	_, messages, err := convertMessages([]core.Message{{Role: "assistant", ExtraFields: fields}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(messages) != 0 {
+		t.Fatalf("empty assistant created a cache-only message: %+v", messages)
+	}
+}
+
+type testBedrockAPIError struct{ code, message string }
+
+func (e testBedrockAPIError) Error() string        { return e.code + ": " + e.message }
+func (e testBedrockAPIError) ErrorCode() string    { return e.code }
+func (e testBedrockAPIError) ErrorMessage() string { return e.message }
+
+func TestCachePointFallbackIsNarrowAndLossless(t *testing.T) {
+	if !isCachePointValidationError(testBedrockAPIError{"ValidationException", "cache point below minimum tokens"}) {
+		t.Fatal("cache-point validation error was not recognized")
+	}
+	if isCachePointValidationError(testBedrockAPIError{"ValidationException", "invalid tool schema"}) {
+		t.Fatal("unrelated validation error would trigger a retry")
+	}
+	parts := converseParts{
+		system: []brtypes.SystemContentBlock{
+			&brtypes.SystemContentBlockMemberText{Value: "system"},
+			&brtypes.SystemContentBlockMemberCachePoint{Value: brtypes.CachePointBlock{Type: brtypes.CachePointTypeDefault}},
+		},
+		messages: []brtypes.Message{{Role: brtypes.ConversationRoleUser, Content: []brtypes.ContentBlock{
+			&brtypes.ContentBlockMemberText{Value: "user"},
+			&brtypes.ContentBlockMemberCachePoint{Value: brtypes.CachePointBlock{Type: brtypes.CachePointTypeDefault}},
+		}}},
+	}
+	clean := withoutCachePoints(parts)
+	if partsHaveCachePoints(clean) || len(clean.system) != 1 || len(clean.messages) != 1 || len(clean.messages[0].Content) != 1 {
+		t.Fatalf("cache-point removal damaged request content: %+v", clean)
+	}
+	if !partsHaveCachePoints(parts) {
+		t.Fatal("cache-point removal mutated original parts")
+	}
+}

internal/providers/bedrock/chat.go

@@ -2,6 +2,7 @@ package bedrock
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math"
 	"strings"
@@ -32,20 +33,86 @@ func (p *Provider) ChatCompletion(ctx context.Context, req *core.ChatRequest) (*
 		return nil, err
 	}
 
-	out, err := p.runtime.Converse(ctx, &bedrockruntime.ConverseInput{
-		ModelId:         parts.modelID,
-		Messages:        parts.messages,
-		System:          parts.system,
-		InferenceConfig: parts.infCfg,
-		ToolConfig:      parts.toolCfg,
-	})
+	out, err := p.runtime.Converse(ctx, converseInput(parts))
+	if err != nil && partsHaveCachePoints(parts) && isCachePointValidationError(err) {
+		parts = withoutCachePoints(parts)
+		out, err = p.runtime.Converse(ctx, converseInput(parts))
+	}
 	if err != nil {
 		return nil, mapAWSError(err)
 	}
 
 	return convertConverseOutput(req.Model, out), nil
 }
 
+func converseInput(parts converseParts) *bedrockruntime.ConverseInput {
+	return &bedrockruntime.ConverseInput{
+		ModelId: parts.modelID, Messages: parts.messages, System: parts.system,
+		InferenceConfig: parts.infCfg, ToolConfig: parts.toolCfg,
+	}
+}
+
+func partsHaveCachePoints(parts converseParts) bool {
+	for _, block := range parts.system {
+		if _, ok := block.(*brtypes.SystemContentBlockMemberCachePoint); ok {
+			return true
+		}
+	}
+	for _, message := range parts.messages {
+		for _, block := range message.Content {
+			if _, ok := block.(*brtypes.ContentBlockMemberCachePoint); ok {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// withoutCachePoints makes the planner best-effort. If a model or regional
+// endpoint rejects Bedrock cache points, the request is retried once without
+// gateway-generated markers rather than failing an otherwise valid call.
+func withoutCachePoints(parts converseParts) converseParts {
+	clean := parts
+	clean.system = make([]brtypes.SystemContentBlock, 0, len(parts.system))
+	for _, block := range parts.system {
+		if _, ok := block.(*brtypes.SystemContentBlockMemberCachePoint); !ok {
+			clean.system = append(clean.system, block)
+		}
+	}
+	clean.messages = make([]brtypes.Message, 0, len(parts.messages))
+	for _, message := range parts.messages {
+		copyMessage := message
+		copyMessage.Content = make([]brtypes.ContentBlock, 0, len(message.Content))
+		for _, block := range message.Content {
+			if _, ok := block.(*brtypes.ContentBlockMemberCachePoint); !ok {
+				copyMessage.Content = append(copyMessage.Content, block)
+			}
+		}
+		if len(copyMessage.Content) > 0 {
+			clean.messages = append(clean.messages, copyMessage)
+		}
+	}
+	return clean
+}
+
+func isCachePointValidationError(err error) bool {
+	type apiError interface {
+		error
+		ErrorCode() string
+		ErrorMessage() string
+	}
+	var apiErr apiError
+	code, message := "", err.Error()
+	if errors.As(err, &apiErr) {
+		code, message = apiErr.ErrorCode(), apiErr.ErrorMessage()
+	}
+	code = strings.ToLower(code)
+	message = strings.ToLower(message)
+	validation := strings.Contains(code, "validation") || strings.Contains(code, "badrequest")
+	return validation && strings.Contains(message, "cache") &&
+		(strings.Contains(message, "point") || strings.Contains(message, "minimum") || strings.Contains(message, "token"))
+}
+
 // converseParts holds the shared pieces of a Converse / ConverseStream request
 // so the two API entry points can stay otherwise independent.
 type converseParts struct {
@@ -153,6 +220,9 @@ func convertMessages(messages []core.Message) ([]brtypes.SystemContentBlock, []b
 				continue
 			}
 			system = append(system, &brtypes.SystemContentBlockMemberText{Value: text})
+			if isGatewayCachePoint(msg.ExtraFields) {
+				system = append(system, &brtypes.SystemContentBlockMemberCachePoint{Value: brtypes.CachePointBlock{Type: brtypes.CachePointTypeDefault}})
+			}
 		case "tool":
 			block, err := convertToolResultMessage(msg)
 			if err != nil {
@@ -164,13 +234,21 @@ func convertMessages(messages []core.Message) ([]brtypes.SystemContentBlock, []b
 			if text == "" {
 				continue
 			}
-			appendOrMerge(brtypes.ConversationRoleUser,
-				[]brtypes.ContentBlock{&brtypes.ContentBlockMemberText{Value: text}})
+			blocks := []brtypes.ContentBlock{&brtypes.ContentBlockMemberText{Value: text}}
+			if isGatewayCachePoint(msg.ExtraFields) {
+				blocks = append(blocks, &brtypes.ContentBlockMemberCachePoint{Value: brtypes.CachePointBlock{Type: brtypes.CachePointTypeDefault}})
+			}
+			appendOrMerge(brtypes.ConversationRoleUser, blocks)
 		case "assistant":
 			blocks, err := convertAssistantMessage(msg)
 			if err != nil {
 				return nil, nil, err
 			}
+			// A cache point is a boundary after real content, never content by
+			// itself. Bedrock rejects cache-point-only messages.
+			if len(blocks) > 0 && isGatewayCachePoint(msg.ExtraFields) {
+				blocks = append(blocks, &brtypes.ContentBlockMemberCachePoint{Value: brtypes.CachePointBlock{Type: brtypes.CachePointTypeDefault}})
+			}
 			appendOrMerge(brtypes.ConversationRoleAssistant, blocks)
 		default:
 			return nil, nil, core.NewInvalidRequestError("unsupported message role: "+msg.Role, nil)
@@ -181,6 +259,11 @@ func convertMessages(messages []core.Message) ([]brtypes.SystemContentBlock, []b
 	return system, out, nil
 }
 
+func isGatewayCachePoint(fields core.UnknownJSONFields) bool {
+	var enabled bool
+	return json.Unmarshal(fields.Lookup(core.GatewayCachePointField), &enabled) == nil && enabled
+}
+
 func convertAssistantMessage(msg core.Message) ([]brtypes.ContentBlock, error) {
 	blocks := make([]brtypes.ContentBlock, 0, 1+len(msg.ToolCalls))
 	if text := core.ExtractTextContent(msg.Content); text != "" {

internal/providers/bedrock/chat_stream.go

@@ -29,20 +29,25 @@ func (p *Provider) StreamChatCompletion(ctx context.Context, req *core.ChatReque
 		return nil, err
 	}
 
-	out, err := p.runtime.ConverseStream(ctx, &bedrockruntime.ConverseStreamInput{
-		ModelId:         parts.modelID,
-		Messages:        parts.messages,
-		System:          parts.system,
-		InferenceConfig: parts.infCfg,
-		ToolConfig:      parts.toolCfg,
-	})
+	out, err := p.runtime.ConverseStream(ctx, converseStreamInput(parts))
+	if err != nil && partsHaveCachePoints(parts) && isCachePointValidationError(err) {
+		parts = withoutCachePoints(parts)
+		out, err = p.runtime.ConverseStream(ctx, converseStreamInput(parts))
+	}
 	if err != nil {
 		return nil, mapAWSError(err)
 	}
 
 	return newOpenAIStream(out, req.Model), nil
 }
 
+func converseStreamInput(parts converseParts) *bedrockruntime.ConverseStreamInput {
+	return &bedrockruntime.ConverseStreamInput{
+		ModelId: parts.modelID, Messages: parts.messages, System: parts.system,
+		InferenceConfig: parts.infCfg, ToolConfig: parts.toolCfg,
+	}
+}
+
 // streamConverter consumes a Bedrock ConverseStream event channel and emits
 // OpenAI-compatible SSE chunks via io.Reader. Reads are buffered: each loop
 // iteration consumes events until at least one chunk's worth of data is ready

internal/providers/cache_control.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"slices"
-	"strings"
 
 	"github.com/goccy/go-json"
 
@@ -74,12 +73,7 @@ func adaptAnthropicBatchCacheControl(ctx context.Context, req *core.BatchRequest
 }
 
 func providerAcceptsAnthropicCacheControl(providerType string) bool {
-	switch strings.ToLower(strings.TrimSpace(providerType)) {
-	case "anthropic", "openrouter":
-		return true
-	default:
-		return false
-	}
+	return promptCacheProfileFor(providerType).acceptsAnthropicCacheControl
 }
 
 func hasAnthropicCacheControl(req *core.ChatRequest) bool {