From 8045a0971da00a515a7036165b1cd5b611aac55e Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Mon, 31 Aug 2026 01:26:23 -0600 Subject: [PATCH 1/3] perf(coding-agent): keep the prompt-cache prefix stable Continual harness state and the current date were serialized into the cached system prompt, so any rlm.harness.create_* write or a midnight date flip changed the prefix and cold-cached a long-running session. A tool-registry refresh could also reorder tool definitions, moving the cache_control marker off the last tool. Volatile content now travels in Context.volatileContext. Anthropic and OpenAI-completions place it after their final cache breakpoint, other providers receive it appended to the message list, and each tool is pinned to the slot it first occupied so an unchanged tool set serializes to identical bytes. The section text the model sees is unchanged; only its position moved. fixes #26 --- .pylon/features.yaml | 19 + .pylon/upstream-review.md | 12 + .../.changes/stable-prompt-cache-prefix.md | 1 + packages/agent/src/agent-loop.ts | 1 + packages/agent/src/agent.ts | 3 + packages/agent/src/types.ts | 11 + .../ai/.changes/stable-prompt-cache-prefix.md | 1 + packages/ai/README.md | 20 + packages/ai/src/api-registry.ts | 20 +- packages/ai/src/providers/anthropic.ts | 32 +- packages/ai/src/providers/faux.ts | 8 +- .../ai/src/providers/openai-completions.ts | 35 ++ .../ai/src/providers/register-builtins.ts | 2 + packages/ai/src/types.ts | 10 + packages/ai/src/utils/volatile-context.ts | 29 ++ .../volatile-context-cache-prefix.test.ts | 379 ++++++++++++++++++ .../.changes/stable-prompt-cache-prefix.md | 2 + .../coding-agent/src/core/agent-session.ts | 37 +- .../coding-agent/src/core/system-prompt.ts | 96 +++-- .../test/agent-session-recursion.test.ts | 10 +- .../26-stable-cache-prefix.test.ts | 228 +++++++++++ .../coding-agent/test/system-prompt.test.ts | 173 ++++---- 22 files changed, 1002 insertions(+), 127 deletions(-) create mode 100644 packages/agent/.changes/stable-prompt-cache-prefix.md create mode 100644 packages/ai/.changes/stable-prompt-cache-prefix.md create mode 100644 packages/ai/src/utils/volatile-context.ts create mode 100644 packages/ai/test/volatile-context-cache-prefix.test.ts create mode 100644 packages/coding-agent/.changes/stable-prompt-cache-prefix.md create mode 100644 packages/coding-agent/test/suite/regressions/26-stable-cache-prefix.test.ts diff --git a/.pylon/features.yaml b/.pylon/features.yaml index 7c09a6582e..03f37fe454 100644 --- a/.pylon/features.yaml +++ b/.pylon/features.yaml @@ -316,3 +316,22 @@ decisions: revisit_when: - Prime makes a user-facing turn abort cancel the child runs it owns, or exposes an equivalent turn-scoped cancellation contract. - Prime introduces a real detached or background spawn mode, which would need a per-child survival decision instead of one retained-versus-active rule. + + stable-prompt-cache-prefix: + area: prompt-ownership + state: shipped + owner: shared + decision: retain + pylon_refs: + - https://github.com/pylon-code/prime-agent/issues/22 + - https://github.com/pylon-code/prime-agent/issues/26 + - https://github.com/pylon-code/prime-agent/pull/32 + upstream_refs: + - https://github.com/PrimeIntellect-ai/prime-agent/commit/4b9e6006fad553dcca5aa69b23a039088064589f + - https://github.com/PrimeIntellect-ai/prime-agent/commit/f81acc6679ab214f1c40821588d9e5e97c47bdb1 + - https://github.com/PrimeIntellect-ai/prime-agent/tree/c382f09856d4a8c8d2b765179657047d58691f25 + fork_change: stable-prompt-cache-prefix-v1 + upstream_support: Prime through c382f09856d4 reduced the system-prompt date to a day stamp for cache stability, but it still serializes mutable continual-harness state and that date into the cached system prompt, has no provider-neutral volatile-content channel, and does not pin active tool order behind the `cache_control` marker on the last tool definition. + revisit_when: + - Prime upstream provides an equivalent provider-neutral channel that keeps mutable state and clock-derived content out of the cached tools/system/history prefix. + - Pylon can drop the fork behavior without regressing measured cache-hit rates through a Claude-Max proxy. diff --git a/.pylon/upstream-review.md b/.pylon/upstream-review.md index 335ac39ed7..b91cffc759 100644 --- a/.pylon/upstream-review.md +++ b/.pylon/upstream-review.md @@ -148,3 +148,15 @@ This ledger records Prime upstream evidence and the decision taken for each over - Validation: new faux-provider regression `packages/coding-agent/test/suite/regressions/25-request-abort-rlm-cascade.test.ts` (2 tests) proves the child's provider stream is cut, that no further child request reaches the provider, that the parent settles and its next turn runs, and pins retained-child survival; it fails on the pre-change implementation. The upstream characterization test was inverted to "cancels active rlm children when the parent turn is interrupted", and the ACP-close terminal-notice retention test now settles the child before the scheduler cut, since a live child no longer survives it. `test/agent-session-recursion.test.ts` passes 112/112 and a 17-file affected batch across abort, RLM, subagent, queue, prompt, compaction, ACP, and correlated-lifecycle suites passes 396/396. `npm run check` is clean. - Fork change: [pylon-code/prime-agent#35](https://github.com/pylon-code/prime-agent/pull/35). - Revisit when Prime makes a user-facing turn abort cancel the children it owns, or introduces a real detached/background spawn mode that needs a per-child survival decision. + +## 2026-08-31 — stable prompt-cache prefix + +- Upstream baseline reviewed: `PrimeIntellect-ai/prime-agent@c382f09856d4a8c8d2b765179657047d58691f25` (one commit past the audited `a903d4b6768f`; PR #1893 renders Mermaid diagrams and does not touch prompt assembly or provider payloads). Latest audited compatibility release remains stock `v0.8.1`. This is a product candidate and does not advance `reviewed_upstream_commit`. +- Searched upstream history and current source for cache-prefix stability work. Prime already recognized the problem for the clock: `4b9e6006f` reduced `Current date and time` to an ISO day stamp and `f81acc667` replaced `toISOString` with local date parts, both to stop the system prompt changing more often than necessary. Nothing else exists: upstream `buildSystemPrompt` still serializes `harnessState` into the cached system prompt, upstream has no `Context.volatileContext` or equivalent provider-neutral channel, and `convertTools` still marks the last tool definition with `cache_control` without pinning active tool order. +- `stable-prompt-cache-prefix`: **retain** a Pylon-owned split. Adopt upstream's day-granularity date decision as-is; it is already in the fork. Add the missing piece upstream lacks: the volatile content leaves the cached region entirely instead of merely changing less often. +- Design: `Context.volatileContext` carries continual-harness state and the current date. The Anthropic provider appends it as the final content block after the `cache_control` breakpoint on the last history block, so the cached tools -> system -> history prefix is unchanged by a harness write or a date flip. The OpenAI-completions provider appends it after its Anthropic-style markers and after the automatically cached prefix of providers without markers. The API registry folds the content into the end of the message list for every other provider, so no provider can silently drop it. `AgentSession` pins each tool to the slot it first occupied, so a registry refresh or an extension reload serializes an unchanged tool set to identical bytes. +- Behavior preservation: the harness-state section and the `Current date` line are byte-identical to what the system prompt previously inlined, including the empty-state rendering and the custom-prompt-only scope of the date. Only their position changed. The subagent guidance still precedes the harness-state roster in the assembled request. +- Compatibility classification: **backward-compatible and additive**. `Context.volatileContext`, `AgentState.volatileContext`, `AgentLoopConfig.getVolatileContext`, and `ApiProvider.handlesVolatileContext` are all optional; a caller that sets none behaves exactly as before. No daemon command, event, protocol version, or schema revision changes. +- Validation: 10 new `packages/ai` payload tests assert a byte-identical cached prefix across volatile changes for both the Anthropic and OpenAI-completions payload builders, correct marker placement, the trailing-user-message case, blank-input handling, and the registry fallback. 4 new `packages/coding-agent/test/suite/regressions/26-stable-cache-prefix.test.ts` cases assert an unchanged prefix across a mid-session harness memory write, an unchanged prefix across a mocked two-day clock advance, identical tool bytes across repeated registry refreshes, and first-activation tool-order pinning across reordering and removal. `test/system-prompt.test.ts` (25), the compaction, compact-skill, refine, runtime, queue, and prompt suites (357), `packages/agent` (60), and 13 focused `packages/ai` provider files (93 passed, 9 gated skips) are green. `npm run check` passes. `test/extensions-runner.test.ts` fails 21 cases identically on unmodified `origin/pylon`, so it is pre-existing and unrelated. +- Deferred: the optional debug-level prefix-drift check from issue #26 is not implemented. The regression tests assert the invariant directly, and a cross-turn byte comparison would need new mutable state inside otherwise pure prompt assembly. +- Revisit when Prime upstream offers an equivalent provider-neutral channel that keeps mutable state and clock-derived content out of the cached prefix, and Pylon can drop the fork behavior without regressing measured cache-hit rates through a Claude-Max proxy. diff --git a/packages/agent/.changes/stable-prompt-cache-prefix.md b/packages/agent/.changes/stable-prompt-cache-prefix.md new file mode 100644 index 0000000000..fbe6abaf63 --- /dev/null +++ b/packages/agent/.changes/stable-prompt-cache-prefix.md @@ -0,0 +1 @@ +- Added `AgentState.volatileContext`, resolved before each model request and forwarded to the provider outside the cached prompt prefix ([#26](https://github.com/pylon-code/prime-agent/issues/26)). diff --git a/packages/agent/src/agent-loop.ts b/packages/agent/src/agent-loop.ts index 31b3a15ccc..c5df50157e 100644 --- a/packages/agent/src/agent-loop.ts +++ b/packages/agent/src/agent-loop.ts @@ -487,6 +487,7 @@ async function streamAssistantResponse( const llmContext: Context = { systemPrompt: config.getSystemPrompt?.() ?? context.systemPrompt, + volatileContext: config.getVolatileContext?.() ?? context.volatileContext, messages: llmMessages, tools: context.tools, }; diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 9c7960bc4a..95c3dcfb66 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -72,6 +72,7 @@ function createMutableAgentState( return { systemPrompt: initialState?.systemPrompt ?? "", + volatileContext: initialState?.volatileContext, model: initialState?.model ?? DEFAULT_MODEL, thinkingLevel: initialState?.thinkingLevel ?? "off", serviceTier: initialState?.serviceTier ?? "default", @@ -451,6 +452,7 @@ export class Agent { private createContextSnapshot(): AgentContext { return { systemPrompt: this._state.systemPrompt, + volatileContext: this._state.volatileContext, messages: this._state.messages.slice(), tools: this._state.tools.slice(), }; @@ -475,6 +477,7 @@ export class Agent { convertToLlm: this.convertToLlm, transformContext: this.transformContext, getSystemPrompt: () => this._state.systemPrompt, + getVolatileContext: () => this._state.volatileContext, getApiKey: this.getApiKey, getSteeringMessages: async () => { if (skipInitialSteeringPoll) { diff --git a/packages/agent/src/types.ts b/packages/agent/src/types.ts index f29074abe0..d592d2a1c6 100644 --- a/packages/agent/src/types.ts +++ b/packages/agent/src/types.ts @@ -171,6 +171,9 @@ export interface AgentLoopConfig extends SimpleStreamOptions { /** Resolves the system prompt immediately before each LLM call. */ getSystemPrompt?: () => string; + /** Resolves the volatile, never-cached context immediately before each LLM call. */ + getVolatileContext?: () => string | undefined; + /** * Resolves an API key dynamically for each LLM call. * @@ -306,6 +309,12 @@ export type AgentMessage = Message | CustomAgentMessages[keyof CustomAgentMessag export interface AgentState { /** System prompt sent with each model request. */ systemPrompt: string; + /** + * Content the model needs but that must stay out of every cached prefix, + * such as mutable harness state or the current date. Providers place it after + * their final prompt-cache breakpoint. + */ + volatileContext?: string; /** Model used for future turns. */ model: Model; /** Requested reasoning level for future turns. */ @@ -374,6 +383,8 @@ export interface AgentTool, @@ -24,6 +25,12 @@ export interface ApiProvider; streamSimple: StreamFunction; + /** + * Set when the provider positions `Context.volatileContext` itself, after its + * final prompt-cache breakpoint. Providers that leave this unset receive the + * volatile content folded into the end of `Context.messages`. + */ + handlesVolatileContext?: boolean; } interface ApiProviderInternal { @@ -42,24 +49,28 @@ const apiProviderRegistry = new Map(); function wrapStream( api: TApi, stream: StreamFunction, + handlesVolatileContext: boolean, ): ApiStreamFunction { return (model, context, options) => { if (model.api !== api) { throw new Error(`Mismatched api: ${model.api} expected ${api}`); } - return stream(model as Model, context, options as TOptions); + const resolved = handlesVolatileContext ? context : foldVolatileContext(context); + return stream(model as Model, resolved, options as TOptions); }; } function wrapStreamSimple( api: TApi, streamSimple: StreamFunction, + handlesVolatileContext: boolean, ): ApiStreamSimpleFunction { return (model, context, options) => { if (model.api !== api) { throw new Error(`Mismatched api: ${model.api} expected ${api}`); } - return streamSimple(model as Model, context, options); + const resolved = handlesVolatileContext ? context : foldVolatileContext(context); + return streamSimple(model as Model, resolved, options); }; } @@ -67,11 +78,12 @@ export function registerApiProvider, sourceId?: string, ): void { + const handlesVolatileContext = provider.handlesVolatileContext === true; apiProviderRegistry.set(provider.api, { provider: { api: provider.api, - stream: wrapStream(provider.api, provider.stream), - streamSimple: wrapStreamSimple(provider.api, provider.streamSimple), + stream: wrapStream(provider.api, provider.stream, handlesVolatileContext), + streamSimple: wrapStreamSimple(provider.api, provider.streamSimple, handlesVolatileContext), }, sourceId, }); diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts index fd01db03ed..12c18a85f2 100644 --- a/packages/ai/src/providers/anthropic.ts +++ b/packages/ai/src/providers/anthropic.ts @@ -42,6 +42,7 @@ import { streamFailureMessage, truncateRawPayload, } from "../utils/stream-failure.js"; +import { resolveVolatileContext } from "../utils/volatile-context.js"; import { resolveCloudflareBaseUrl } from "./cloudflare.js"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.js"; @@ -965,9 +966,15 @@ function buildParams( options?: AnthropicOptions, cacheControl?: CacheControlEphemeral, ): MessageCreateParamsStreaming { + const messages = convertMessages(context.messages, model, isOAuthToken, cacheControl); + // Anthropic caches the prefix in tools -> system -> messages order, and the last + // breakpoint sits on the final history block. Appending volatile content after + // that breakpoint keeps a harness-state or date change out of every cached region. + appendVolatileContext(messages, resolveVolatileContext(context)); + const params: MessageCreateParamsStreaming = { model: model.id, - messages: convertMessages(context.messages, model, isOAuthToken, cacheControl), + messages, max_tokens: options?.maxTokens || (model.maxTokens / 3) | 0, stream: true, }; @@ -1063,6 +1070,29 @@ function buildParams( return params; } +/** + * Append volatile content as the final block of the request, after any + * `cache_control` breakpoint placed by {@link convertMessages}. + */ +function appendVolatileContext(messages: MessageParam[], volatileContext: string | undefined): void { + if (!volatileContext) return; + + const text = sanitizeSurrogates(volatileContext); + const lastMessage = messages[messages.length - 1]; + if (lastMessage?.role === "user") { + lastMessage.content = + typeof lastMessage.content === "string" + ? [ + { type: "text", text: lastMessage.content }, + { type: "text", text }, + ] + : [...lastMessage.content, { type: "text", text }]; + return; + } + + messages.push({ role: "user", content: [{ type: "text", text }] }); +} + // Normalize tool call IDs to match Anthropic's required pattern and length function normalizeToolCallId(id: string): string { return id.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64); diff --git a/packages/ai/src/providers/faux.ts b/packages/ai/src/providers/faux.ts index 9bed351325..bdc28f7631 100644 --- a/packages/ai/src/providers/faux.ts +++ b/packages/ai/src/providers/faux.ts @@ -203,6 +203,10 @@ function serializeContext(context: Context): string { if (context.tools?.length) { parts.push(`tools:${JSON.stringify(context.tools)}`); } + // Last, so the simulated cache prefix stays stable when volatile content changes. + if (context.volatileContext) { + parts.push(`volatile:${context.volatileContext}`); + } return parts.join("\n\n"); } @@ -494,7 +498,9 @@ export function registerFauxProvider(options: RegisterFauxProviderOptions = {}): const streamSimple: StreamFunction = (streamModel, context, streamOptions) => stream(streamModel, context, streamOptions); - registerApiProvider({ api, stream, streamSimple }, sourceId); + // Volatile context stays on the context so tests observe exactly what the caller + // assembled, instead of a provider-specific folded message. + registerApiProvider({ api, stream, streamSimple, handlesVolatileContext: true }, sourceId); function getModel(): Model; function getModel(requestedModelId: string): Model | undefined; diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index 00f945f17b..113136ee01 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -35,6 +35,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream.js"; import { headersToRecord } from "../utils/headers.js"; import { parseStreamingJson } from "../utils/json-parse.js"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; +import { resolveVolatileContext } from "../utils/volatile-context.js"; import { isCloudflareProvider, resolveCloudflareBaseUrl } from "./cloudflare.js"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.js"; import { buildBaseOptions } from "./simple-options.js"; @@ -624,6 +625,10 @@ function buildParams( applyAnthropicCacheControl(messages, params.tools, cacheControl); } + // Appended after the cache markers (and after the automatically cached prefix of + // providers without markers) so volatile content cannot cold-cache the request. + appendVolatileContext(messages, resolveVolatileContext(context), compat); + if (options?.toolChoice) { params.tool_choice = options.toolChoice; } @@ -696,6 +701,36 @@ function getCompatCacheControl( return { type: "ephemeral", ...(ttl ? { ttl } : {}) }; } +/** + * Append volatile content as the last thing the request carries, after any + * cache markers applied by {@link applyAnthropicCacheControl}. + */ +function appendVolatileContext( + messages: ChatCompletionMessageParam[], + volatileContext: string | undefined, + compat: ResolvedOpenAICompletionsCompat, +): void { + if (!volatileContext) return; + + const text = sanitizeSurrogates(volatileContext); + const lastMessage = messages[messages.length - 1]; + if (lastMessage?.role === "user") { + lastMessage.content = + typeof lastMessage.content === "string" + ? [ + { type: "text", text: lastMessage.content }, + { type: "text", text }, + ] + : [...lastMessage.content, { type: "text", text }]; + return; + } + + if (compat.requiresAssistantAfterToolResult && lastMessage?.role === "tool") { + messages.push({ role: "assistant", content: "I have processed the tool results." }); + } + messages.push({ role: "user", content: text }); +} + function applyAnthropicCacheControl( messages: ChatCompletionMessageParam[], tools: OpenAI.Chat.Completions.ChatCompletionTool[] | undefined, diff --git a/packages/ai/src/providers/register-builtins.ts b/packages/ai/src/providers/register-builtins.ts index 6447568084..96fb2596ed 100644 --- a/packages/ai/src/providers/register-builtins.ts +++ b/packages/ai/src/providers/register-builtins.ts @@ -344,12 +344,14 @@ export function registerBuiltInApiProviders(): void { api: "anthropic-messages", stream: streamAnthropic, streamSimple: streamSimpleAnthropic, + handlesVolatileContext: true, }); registerApiProvider({ api: "openai-completions", stream: streamOpenAICompletions, streamSimple: streamSimpleOpenAICompletions, + handlesVolatileContext: true, }); registerApiProvider({ diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 5bb3034865..95da0fda30 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -254,6 +254,16 @@ export interface Context { systemPrompt?: string; messages: Message[]; tools?: Tool[]; + /** + * Content the model still needs but that must never enter a cached prefix, + * such as mutable harness state or the current date. + * + * Providers place it after their final prompt-cache breakpoint so changing it + * cannot invalidate the cached tools, system prompt, or conversation history. + * Providers that do not mark cache breakpoints receive it folded into the end + * of `messages` instead, so the content is never dropped. + */ + volatileContext?: string; } /** diff --git a/packages/ai/src/utils/volatile-context.ts b/packages/ai/src/utils/volatile-context.ts new file mode 100644 index 0000000000..59254a5104 --- /dev/null +++ b/packages/ai/src/utils/volatile-context.ts @@ -0,0 +1,29 @@ +import type { Context, Message, UserMessage } from "../types.js"; + +/** Volatile content for this request with surrounding whitespace removed, or undefined when empty. */ +export function resolveVolatileContext(context: Context): string | undefined { + const text = context.volatileContext?.trim(); + return text ? text : undefined; +} + +/** + * Move volatile content to the end of the message list. + * + * Used for providers that do not place explicit cache breakpoints: the content + * still reaches the model, and because it is last it cannot shift the bytes of + * an automatically cached prefix. + */ +export function foldVolatileContext(context: Context): Context { + const text = resolveVolatileContext(context); + if (!text) { + return context.volatileContext === undefined ? context : { ...context, volatileContext: undefined }; + } + + const trailing: UserMessage = { + role: "user", + content: [{ type: "text", text }], + timestamp: Date.now(), + }; + const messages: Message[] = [...context.messages, trailing]; + return { ...context, messages, volatileContext: undefined }; +} diff --git a/packages/ai/test/volatile-context-cache-prefix.test.ts b/packages/ai/test/volatile-context-cache-prefix.test.ts new file mode 100644 index 0000000000..f6f0cb096b --- /dev/null +++ b/packages/ai/test/volatile-context-cache-prefix.test.ts @@ -0,0 +1,379 @@ +import { Type } from "typebox"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { registerApiProvider, unregisterApiProviders } from "../src/api-registry.js"; +import { getModel } from "../src/models.js"; +import { streamAnthropic } from "../src/providers/anthropic.js"; +import { fauxAssistantMessage, registerFauxProvider } from "../src/providers/faux.js"; +import { streamOpenAICompletions } from "../src/providers/openai-completions.js"; +import { streamSimple } from "../src/stream.js"; +import type { Context, Model, StreamFunction, StreamOptions } from "../src/types.js"; +import { createAssistantMessageEventStream } from "../src/utils/event-stream.js"; + +interface CacheControl { + type: "ephemeral"; + ttl?: string; +} + +interface TextPart { + type: string; + text?: string; + cache_control?: CacheControl; +} + +interface CapturedMessage { + role: string; + content: string | TextPart[] | null; +} + +interface CapturedPayload { + system?: unknown; + tools?: unknown[]; + messages: CapturedMessage[]; +} + +const TOOLS = [ + { + name: "read", + description: "Read a file", + parameters: Type.Object({ path: Type.String() }), + }, + { + name: "write", + description: "Write a file", + parameters: Type.Object({ path: Type.String() }), + }, +]; + +function baseContext(volatileContext?: string): Context { + return { + systemPrompt: "You are a general purpose agent.", + messages: [ + { role: "user", content: "first", timestamp: 1 }, + { + role: "assistant", + content: [{ type: "text", text: "reply" }], + api: "anthropic-messages", + provider: "anthropic", + model: "claude-sonnet-4-5", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: 2, + }, + { role: "user", content: "second", timestamp: 3 }, + ], + tools: TOOLS, + volatileContext, + }; +} + +function flattenParts(message: CapturedMessage): TextPart[] { + const content = message.content; + if (content === null) return []; + if (typeof content === "string") return [{ type: "text", text: content }]; + return content; +} + +/** + * Serialize everything an Anthropic-style request caches: the tool definitions, + * the system blocks, and the history up to and including the last cache marker. + */ +function cachedPrefix(payload: CapturedPayload): string { + const history: string[] = []; + let lastMarker = -1; + for (const message of payload.messages) { + for (const part of flattenParts(message)) { + if (part.cache_control) { + lastMarker = history.length; + } + history.push(JSON.stringify({ role: message.role, part })); + } + } + return JSON.stringify({ + tools: payload.tools, + system: payload.system, + history: history.slice(0, lastMarker + 1), + }); +} + +function lastPart(payload: CapturedPayload): TextPart | undefined { + const parts = flattenParts(payload.messages[payload.messages.length - 1]); + return parts[parts.length - 1]; +} + +async function captureAnthropicPayload(volatileContext?: string): Promise { + const model: Model<"anthropic-messages"> = { + ...getModel("anthropic", "claude-sonnet-4-5"), + baseUrl: "http://127.0.0.1:9", + }; + let captured: CapturedPayload | undefined; + await streamAnthropic(model, baseContext(volatileContext), { + apiKey: "fake-key", + maxRetries: 0, + onPayload: (payload) => { + captured = payload as CapturedPayload; + return payload; + }, + }).result(); + + if (!captured) { + throw new Error("Expected the Anthropic payload to be captured before the request failed"); + } + return captured; +} + +const openAIMockState = vi.hoisted(() => ({ + lastParams: undefined as CapturedPayload | undefined, +})); + +vi.mock("openai", () => { + class FakeOpenAI { + chat = { + completions: { + create: (params: CapturedPayload) => { + openAIMockState.lastParams = params; + const stream = { + async *[Symbol.asyncIterator]() { + yield { + id: "chatcmpl-test", + choices: [{ delta: {}, finish_reason: "stop" }], + }; + }, + }; + const promise = Promise.resolve(stream) as Promise & { + withResponse: () => Promise<{ + data: typeof stream; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: stream, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }, + }; + } + + return { default: FakeOpenAI }; +}); + +const anthropicCompatModel: Model<"openai-completions"> = { + id: "anthropic-proxy", + name: "Anthropic Proxy", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://example.com/v1", + reasoning: false, + input: ["text"], + cost: { input: 1, output: 1, cacheRead: 0, cacheWrite: 1 }, + contextWindow: 128000, + maxTokens: 32000, + compat: { cacheControlFormat: "anthropic" }, +}; + +const plainOpenAIModel: Model<"openai-completions"> = { + ...anthropicCompatModel, + id: "plain-completions", + name: "Plain Completions", + compat: undefined, +}; + +async function captureOpenAIPayload( + model: Model<"openai-completions">, + volatileContext?: string, +): Promise { + openAIMockState.lastParams = undefined; + await streamOpenAICompletions(model, baseContext(volatileContext), { apiKey: "test-key" }).result(); + if (!openAIMockState.lastParams) { + throw new Error("Expected the OpenAI payload to be captured"); + } + return openAIMockState.lastParams; +} + +describe("Anthropic volatile context placement", () => { + it("keeps the cached prefix byte-identical when volatile context changes", async () => { + const first = await captureAnthropicPayload("# Continual Harness State\n\nmemory: 0"); + const second = await captureAnthropicPayload("# Continual Harness State\n\nmemory: 1\n- [local:m1] note"); + + expect(cachedPrefix(second)).toBe(cachedPrefix(first)); + expect(JSON.stringify(second.messages)).not.toBe(JSON.stringify(first.messages)); + }); + + it("keeps the cached prefix byte-identical against a request with no volatile context", async () => { + const withoutVolatile = await captureAnthropicPayload(); + const withVolatile = await captureAnthropicPayload("Current date: 2026-09-01"); + + expect(cachedPrefix(withVolatile)).toBe(cachedPrefix(withoutVolatile)); + }); + + it("places volatile context after the last cache marker", async () => { + const payload = await captureAnthropicPayload("Current date: 2026-09-01"); + const parts = flattenParts(payload.messages[payload.messages.length - 1]); + + expect(parts).toHaveLength(2); + expect(parts[0]?.text).toBe("second"); + expect(parts[0]?.cache_control).toEqual({ type: "ephemeral" }); + expect(parts[1]?.text).toBe("Current date: 2026-09-01"); + expect(parts[1]?.cache_control).toBeUndefined(); + }); + + it("marks the last tool definition and leaves the system prompt free of volatile content", async () => { + const payload = await captureAnthropicPayload("# Continual Harness State\n\nmemory: 1"); + const tools = payload.tools as Array<{ name: string; cache_control?: CacheControl }>; + + expect(tools.map((tool) => tool.name)).toEqual(["read", "write"]); + expect(tools[0]?.cache_control).toBeUndefined(); + expect(tools[1]?.cache_control).toEqual({ type: "ephemeral" }); + expect(JSON.stringify(payload.system)).not.toContain("Continual Harness State"); + }); + + it("appends a trailing user message when the request ends with an assistant turn", async () => { + const model: Model<"anthropic-messages"> = { + ...getModel("anthropic", "claude-sonnet-4-5"), + baseUrl: "http://127.0.0.1:9", + }; + const context = baseContext("volatile tail"); + context.messages = context.messages.slice(0, 2); + let captured: CapturedPayload | undefined; + await streamAnthropic(model, context, { + apiKey: "fake-key", + maxRetries: 0, + onPayload: (payload) => { + captured = payload as CapturedPayload; + return payload; + }, + }).result(); + + if (!captured) throw new Error("Expected the Anthropic payload to be captured"); + const last = captured.messages[captured.messages.length - 1]; + expect(last.role).toBe("user"); + expect(lastPart(captured)?.text).toBe("volatile tail"); + expect(lastPart(captured)?.cache_control).toBeUndefined(); + }); + + it("ignores blank volatile context", async () => { + const payload = await captureAnthropicPayload(" \n "); + expect(flattenParts(payload.messages[payload.messages.length - 1])).toHaveLength(1); + }); +}); + +describe("OpenAI-completions volatile context placement", () => { + beforeEach(() => { + openAIMockState.lastParams = undefined; + }); + + it("keeps the cached prefix byte-identical when volatile context changes", async () => { + const first = await captureOpenAIPayload(anthropicCompatModel, "memory: 0"); + const second = await captureOpenAIPayload(anthropicCompatModel, "memory: 1\n- [local:m1] note"); + + expect(cachedPrefix(second)).toBe(cachedPrefix(first)); + expect(JSON.stringify(second.messages)).not.toBe(JSON.stringify(first.messages)); + }); + + it("places volatile context after the Anthropic-style cache markers", async () => { + const payload = await captureOpenAIPayload(anthropicCompatModel, "memory: 1"); + const parts = flattenParts(payload.messages[payload.messages.length - 1]); + + expect(parts).toHaveLength(2); + expect(parts[0]?.cache_control).toEqual({ type: "ephemeral" }); + expect(parts[1]?.text).toBe("memory: 1"); + expect(parts[1]?.cache_control).toBeUndefined(); + }); + + it("appends volatile context last for providers with automatic prefix caching", async () => { + const payload = await captureOpenAIPayload(plainOpenAIModel, "memory: 1"); + const parts = flattenParts(payload.messages[payload.messages.length - 1]); + + expect(parts.map((part) => part.text)).toEqual(["second", "memory: 1"]); + const instruction = payload.messages.find((message) => message.role === "system"); + expect(JSON.stringify(instruction)).not.toContain("memory: 1"); + }); +}); + +describe("volatile context handling at the provider registry", () => { + function recordingProvider( + api: string, + handlesVolatileContext: boolean, + ): { contexts: Context[]; model: Model } { + const contexts: Context[] = []; + const record: StreamFunction = (_model, context) => { + contexts.push(context); + return createAssistantMessageEventStream(); + }; + registerApiProvider({ api, stream: record, streamSimple: record, handlesVolatileContext }, api); + return { + contexts, + model: { + id: `${api}-model`, + name: api, + api, + provider: api, + baseUrl: "http://localhost:0", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000, + maxTokens: 100, + }, + }; + } + + afterEach(() => { + unregisterApiProviders("fallback-api"); + unregisterApiProviders("native-api"); + }); + + it("folds volatile context into the end of the message list for providers without cache markers", () => { + const { contexts, model } = recordingProvider("fallback-api", false); + + streamSimple(model, baseContext("# Continual Harness State\n\nmemory: 1")); + + expect(contexts).toHaveLength(1); + const received = contexts[0]; + expect(received.volatileContext).toBeUndefined(); + expect(received.systemPrompt).toBe("You are a general purpose agent."); + expect(received.messages).toHaveLength(baseContext().messages.length + 1); + const last = received.messages[received.messages.length - 1]; + expect(last.role).toBe("user"); + expect(JSON.stringify(last)).toContain("Continual Harness State"); + }); + + it("leaves volatile context untouched for providers that place it themselves", () => { + const { contexts, model } = recordingProvider("native-api", true); + + streamSimple(model, baseContext("volatile tail")); + + expect(contexts).toHaveLength(1); + expect(contexts[0].volatileContext).toBe("volatile tail"); + expect(contexts[0].messages).toHaveLength(baseContext().messages.length); + }); + + it("keeps volatile content out of the faux provider's simulated cache prefix", async () => { + const faux = registerFauxProvider(); + try { + const context = baseContext("memory: 0"); + faux.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("second")]); + const options = { sessionId: "cache-session" }; + + await streamSimple(faux.getModel(), context, options).result(); + const second = await streamSimple( + faux.getModel(), + { ...context, volatileContext: "memory: 1\n- [local:m1] note" }, + options, + ).result(); + + expect(second.usage.cacheRead).toBeGreaterThan(0); + expect(second.usage.cacheWrite).toBeLessThan(second.usage.cacheRead); + } finally { + faux.unregister(); + } + }); +}); diff --git a/packages/coding-agent/.changes/stable-prompt-cache-prefix.md b/packages/coding-agent/.changes/stable-prompt-cache-prefix.md new file mode 100644 index 0000000000..e1deb985ca --- /dev/null +++ b/packages/coding-agent/.changes/stable-prompt-cache-prefix.md @@ -0,0 +1,2 @@ +- Fixed prompt-cache churn by sending continual harness state and the current date outside the cached prompt prefix, so a harness refinement or a midnight date flip no longer re-caches a long-running session ([#26](https://github.com/pylon-code/prime-agent/issues/26)). +- Pinned each tool to the slot it first occupied so a tool-registry refresh serializes an unchanged tool set to identical bytes ([#26](https://github.com/pylon-code/prime-agent/issues/26)). diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index c3d07b21d9..e9d9f5e71e 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -282,7 +282,7 @@ import { type SlashCommandInfo, } from "./slash-commands.js"; import { createSyntheticSourceInfo, type SourceInfo } from "./source-info.js"; -import { type BuildSystemPromptOptions, buildSystemPrompt } from "./system-prompt.js"; +import { type BuildSystemPromptOptions, buildSystemPrompt, buildVolatileContext } from "./system-prompt.js"; import { THINKING_LEVELS } from "./thinking-levels.js"; import { type BashOperations, createLocalBashOperations } from "./tools/bash.js"; import { createAllToolDefinitions } from "./tools/index.js"; @@ -1233,6 +1233,8 @@ export class AgentSession { private _toolDefinitions: Map = new Map(); private _toolPromptSnippets: Map = new Map(); private _toolPromptGuidelines: Map = new Map(); + /** First-activation slot per tool name; keeps the serialized tool order stable. */ + private readonly _toolOrderSlots: Map = new Map(); private _baseSystemPrompt = ""; private _baseSystemPromptOptions!: BuildSystemPromptOptions; @@ -4510,9 +4512,22 @@ export class AgentSession { return this._toolDefinitions.get(name)?.definition; } + /** + * Slot a tool name occupies in the serialized tool list, assigned the first + * time the session activates it. + */ + private _toolOrderSlot(name: string): number { + const existing = this._toolOrderSlots.get(name); + if (existing !== undefined) { + return existing; + } + const slot = this._toolOrderSlots.size; + this._toolOrderSlots.set(name, slot); + return slot; + } + setActiveToolsByName(toolNames: string[]): void { - const tools: AgentTool[] = []; - const validToolNames: string[] = []; + const selected: Array<{ tool: AgentTool; name: string; slot: number }> = []; const seenToolNames = new Set(); for (const name of toolNames) { if (seenToolNames.has(name)) { @@ -4521,13 +4536,17 @@ export class AgentSession { const tool = this._toolRegistry.get(name); if (tool) { seenToolNames.add(name); - tools.push(tool); - validToolNames.push(name); + selected.push({ tool, name, slot: this._toolOrderSlot(name) }); } } - this.agent.state.tools = tools; + // Pin each tool to the slot it first occupied. A registry refresh, an + // extension reload, or a removed-then-restored tool then serializes the same + // tool set to the same bytes, so the provider's cache breakpoint on the last + // tool definition survives. + selected.sort((left, right) => left.slot - right.slot); + this.agent.state.tools = selected.map((entry) => entry.tool); - this._baseSystemPrompt = this._rebuildSystemPrompt(validToolNames); + this._baseSystemPrompt = this._rebuildSystemPrompt(selected.map((entry) => entry.name)); this.agent.state.systemPrompt = this._baseSystemPrompt; } @@ -4699,6 +4718,10 @@ export class AgentSession { harnessState: this._loadMergedHarnessState(), genericMcpServers: this._mcpManager?.getEnabledGenericServers(), }; + // Harness state and the current date are refreshed here alongside the stable + // prompt, but travel separately so a refine or a date flip cannot change the + // bytes of the provider's cached tools/system/history prefix. + this.agent.state.volatileContext = buildVolatileContext(this._baseSystemPromptOptions); return buildSystemPrompt(this._baseSystemPromptOptions); } diff --git a/packages/coding-agent/src/core/system-prompt.ts b/packages/coding-agent/src/core/system-prompt.ts index a4cc0ec236..1232288c60 100644 --- a/packages/coding-agent/src/core/system-prompt.ts +++ b/packages/coding-agent/src/core/system-prompt.ts @@ -37,7 +37,66 @@ export interface BuildSystemPromptOptions { genericMcpServers?: string[]; } -/** Build the system prompt with tools, guidelines, and context */ +/** Current-day stamp, deliberately without a time component. */ +function currentDate(): string { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, "0"); + const day = String(now.getDate()).padStart(2, "0"); + return `${year}-${month}-${day}`; +} + +/** Tool- and skill-derived facts shared by the stable prompt and the volatile block. */ +function resolvePromptScope(options: BuildSystemPromptOptions) { + const skills = options.skills ?? []; + const tools = options.selectedTools ?? ["ipython"]; + const visibleSkills = skills.filter((skill) => !skill.disableModelInvocation); + return { + skills, + tools, + hasIpython: tools.includes("ipython"), + hasBash: tools.includes("bash"), + visibleSkills, + visiblePythonSkillImportNames: getPythonSkillRuntimeInfo(visibleSkills).map((skill) => skill.importName), + hasRefineSkill: visibleSkills.some((skill) => skill.name === REFINE_SKILL_NAME), + }; +} + +/** + * Build the volatile prompt block: content the model must see but that changes + * for reasons unrelated to the request, so it cannot live in the cached prefix. + * + * The section text is identical to what the system prompt used to inline; only + * its position moved, to after the provider's last prompt-cache breakpoint. + */ +export function buildVolatileContext(options: BuildSystemPromptOptions): string | undefined { + const { hasIpython, hasBash, hasRefineSkill } = resolvePromptScope(options); + const sections: string[] = []; + + // Only custom prompts ever carried a date line; keep that scope unchanged. + if (options.customPrompt) { + sections.push(`Current date: ${currentDate()}`); + } + + if (options.harnessState) { + sections.push( + formatHarnessStateForPrompt(options.harnessState, { + includeIpythonExamples: hasIpython, + includeShellExamples: hasBash, + includeRefineExamples: hasIpython && hasRefineSkill, + }), + ); + } + + return sections.length > 0 ? sections.join("\n\n") : undefined; +} + +/** + * Build the stable system prompt with tools, guidelines, and context. + * + * Volatile content (harness state, current date) is deliberately excluded; see + * {@link buildVolatileContext}. + */ export function buildSystemPrompt(options: BuildSystemPromptOptions): string { const { customPrompt, @@ -47,29 +106,16 @@ export function buildSystemPrompt(options: BuildSystemPromptOptions): string { cwd, messagesPath, contextFiles: providedContextFiles, - skills: providedSkills, allowRecursion, - harnessState, } = options; const promptCwd = cwd.replace(/\\/g, "/"); const promptMessagesPath = (messagesPath ?? "not persisted").replace(/\\/g, "/"); - const now = new Date(); - const year = now.getFullYear(); - const month = String(now.getMonth() + 1).padStart(2, "0"); - const day = String(now.getDate()).padStart(2, "0"); - const date = `${year}-${month}-${day}`; - const appendSection = appendSystemPrompt ? `\n\n${appendSystemPrompt}` : ""; const contextFiles = providedContextFiles ?? []; - const skills = providedSkills ?? []; - const tools = selectedTools ?? ["ipython"]; - const hasIpython = tools.includes("ipython"); - const hasBash = tools.includes("bash"); - const visibleSkills = skills.filter((skill) => !skill.disableModelInvocation); - const visiblePythonSkillImportNames = getPythonSkillRuntimeInfo(visibleSkills).map((skill) => skill.importName); - const hasRefineSkill = visibleSkills.some((skill) => skill.name === REFINE_SKILL_NAME); + const { skills, tools, hasIpython, visibleSkills, visiblePythonSkillImportNames, hasRefineSkill } = + resolvePromptScope(options); const genericMcpSection = hasIpython ? formatGenericMcpGuidance(options.genericMcpServers) : ""; if (customPrompt) { @@ -91,8 +137,7 @@ export function buildSystemPrompt(options: BuildSystemPromptOptions): string { prompt += formatSkillsForPrompt(skills); } - // Add date and working directory last - prompt += `\nCurrent date: ${date}`; + // Add the working directory last prompt += `\nCurrent working directory: ${promptCwd}`; const childDoctrine = buildChildAgentDoctrine({ @@ -105,10 +150,6 @@ export function buildSystemPrompt(options: BuildSystemPromptOptions): string { prompt += `\n\n${childDoctrine}`; } - if (harnessState) { - prompt += `\n\n${formatHarnessStateForPrompt(harnessState, { includeIpythonExamples: hasIpython, includeShellExamples: hasBash, includeRefineExamples: hasIpython && hasRefineSkill })}`; - } - if (genericMcpSection) { prompt += `\n\n${genericMcpSection}`; } @@ -130,9 +171,10 @@ export function buildSystemPrompt(options: BuildSystemPromptOptions): string { parentAgent: options.rlmParentAgent, }); - // Appended AFTER the trained buildRlmPrompt prefix, and before the harness-state - // menu, so the model reads when/why to delegate and then sees the concrete subagent - // specs it can match against — the same ordering as Claude Code's Agent tool. + // Appended AFTER the trained buildRlmPrompt prefix, so the model reads when/why to + // delegate before it reaches the concrete subagent specs it can match against — the + // same ordering as Claude Code's Agent tool. The specs themselves now arrive in the + // volatile block, which still follows this guidance in the assembled request. if ((allowRecursion ?? true) && hasIpython) { const visiblePythonSkillNames = new Set( getPythonSkillRuntimeInfo(visibleSkills).map((skill) => skill.importName), @@ -144,10 +186,6 @@ export function buildSystemPrompt(options: BuildSystemPromptOptions): string { })}`; } - if (harnessState) { - prompt += `\n\n${formatHarnessStateForPrompt(harnessState, { includeIpythonExamples: hasIpython, includeShellExamples: hasBash, includeRefineExamples: hasIpython && hasRefineSkill })}`; - } - if (genericMcpSection) { prompt += `\n\n${genericMcpSection}`; } diff --git a/packages/coding-agent/test/agent-session-recursion.test.ts b/packages/coding-agent/test/agent-session-recursion.test.ts index e43ce9900e..007edd74c6 100644 --- a/packages/coding-agent/test/agent-session-recursion.test.ts +++ b/packages/coding-agent/test/agent-session-recursion.test.ts @@ -4139,7 +4139,7 @@ describe("AgentSession RLM session dir", () => { expect(env.RLM_HARNESS_STATE_DIR).toBe(join(ephemeralDir, "harness")); }); - it("loads the ephemeral RLM harness path into the host system prompt", () => { + it("loads the ephemeral RLM harness path into the host volatile context", () => { const ephemeralDir = join(tempDir, "ephemeral-rlm"); mkdirSync(join(ephemeralDir, "harness"), { recursive: true }); writeFileSync( @@ -4174,10 +4174,12 @@ describe("AgentSession RLM session dir", () => { ); const root = createSession(SessionManager.inMemory(tempDir), undefined, undefined, false, ephemeralDir); - const prompt = root.systemPrompt; + // Harness state travels outside the cached system prompt. + const volatileContext = root.agent.state.volatileContext ?? ""; - expect(prompt).toContain("Ephemeral note"); - expect(prompt).toContain("Loaded from the RLM session harness path."); + expect(root.systemPrompt).not.toContain("Ephemeral note"); + expect(volatileContext).toContain("Ephemeral note"); + expect(volatileContext).toContain("Loaded from the RLM session harness path."); }); it("exports the configured agentDir to the kernel so skills find auth.json", () => { diff --git a/packages/coding-agent/test/suite/regressions/26-stable-cache-prefix.test.ts b/packages/coding-agent/test/suite/regressions/26-stable-cache-prefix.test.ts new file mode 100644 index 0000000000..33056af14a --- /dev/null +++ b/packages/coding-agent/test/suite/regressions/26-stable-cache-prefix.test.ts @@ -0,0 +1,228 @@ +import { join } from "node:path"; +import type { AgentTool } from "@earendil-works/pi-agent-core"; +import type { Context, Model, StreamOptions } from "@earendil-works/pi-ai"; +import { Type } from "typebox"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { applyRefinementProposal, loadHarnessState, saveHarnessState } from "../../../src/core/refinement/index.js"; +import { createTestResourceLoader } from "../../utilities.js"; +import { createHarness, type Harness, type HarnessOptions } from "../harness.js"; + +/** + * Provider-visible request shape, captured without the non-serializable tool + * executors. `prefix` is everything a provider caches; `volatile` is the block + * that travels after the last cache breakpoint. + */ +interface RequestSnapshot { + prefix: { systemPrompt: string; tools: string; history: string }; + volatile: string; +} + +function serializeTools(tools: Context["tools"]): string { + return JSON.stringify( + (tools ?? []).map((tool) => ({ + name: tool.name, + description: tool.description, + parameters: tool.parameters, + })), + ); +} + +function snapshotRequest(context: Context): RequestSnapshot { + return { + prefix: { + systemPrompt: context.systemPrompt ?? "", + tools: serializeTools(context.tools), + history: JSON.stringify(context.messages), + }, + volatile: context.volatileContext ?? "", + }; +} + +function tool(name: string): AgentTool { + return { + name, + label: name, + description: `${name} description`, + parameters: Type.Object({ input: Type.String() }), + execute: async () => ({ content: [{ type: "text", text: "ok" }], details: undefined }), + }; +} + +async function createRecordingHarness(options: HarnessOptions = {}): Promise<{ + harness: Harness; + requests: RequestSnapshot[]; +}> { + const harness = await createHarness({ + persistSession: true, + tools: [tool("alpha"), tool("beta"), tool("gamma")], + ...options, + }); + const requests: RequestSnapshot[] = []; + const record = (context: Context, _options: StreamOptions | undefined, _state: unknown, model: Model) => { + requests.push(snapshotRequest(context)); + return { + role: "assistant" as const, + content: [{ type: "text" as const, text: "ok" }], + api: harness.faux.api, + provider: "faux", + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop" as const, + timestamp: Date.now(), + }; + }; + harness.setResponses([record, record, record]); + return { harness, requests }; +} + +/** Local-day stamp in the same format the prompt builder emits. */ +function localDate(): string { + const now = new Date(); + const month = String(now.getMonth() + 1).padStart(2, "0"); + const day = String(now.getDate()).padStart(2, "0"); + return `${now.getFullYear()}-${month}-${day}`; +} + +/** Persist a memory the way a kernel-side `rlm.harness.create_memory` call would. */ +function seedLocalMemory(harness: Harness, id: string, content: string): void { + const localDir = join(harness.sessionManager.getSessionArtifactDir() ?? harness.tempDir, "harness"); + const state = loadHarnessState(localDir, "local"); + applyRefinementProposal( + state, + { + summary: `Seed ${id}`, + rationale: "seed", + expectedOutcome: "seeded", + edits: [{ action: "create", kind: "memory", id, title: id, content }], + }, + { id: `refine_${id}`, scope: "local" }, + ); + saveHarnessState(localDir, state); +} + +describe("issue 26: stable prompt-cache prefix", () => { + const harnesses: Harness[] = []; + let previousAgentDir: string | undefined; + let agentDirWasSet = false; + + function isolateGlobalHarnessState(harness: Harness): void { + if (!agentDirWasSet) { + previousAgentDir = process.env.PRIME_AGENT_CODING_AGENT_DIR; + agentDirWasSet = true; + } + process.env.PRIME_AGENT_CODING_AGENT_DIR = join(harness.tempDir, "agent"); + } + + afterEach(() => { + vi.useRealTimers(); + if (agentDirWasSet) { + if (previousAgentDir === undefined) { + delete process.env.PRIME_AGENT_CODING_AGENT_DIR; + } else { + process.env.PRIME_AGENT_CODING_AGENT_DIR = previousAgentDir; + } + agentDirWasSet = false; + previousAgentDir = undefined; + } + while (harnesses.length > 0) { + harnesses.pop()?.cleanup(); + } + }); + + it("keeps the cached prefix byte-identical across a mid-session harness update", async () => { + const { harness, requests } = await createRecordingHarness(); + harnesses.push(harness); + isolateGlobalHarnessState(harness); + + await harness.session.prompt("first"); + seedLocalMemory(harness, "cache_note", "Prompt caching needs a stable prefix."); + // The same rebuild a harness write triggers through the tool-registry refresh. + harness.session.setActiveToolsByName(harness.session.getActiveToolNames()); + await harness.session.prompt("second"); + + expect(requests).toHaveLength(2); + const [first, second] = requests; + + expect(second.prefix.systemPrompt).toBe(first.prefix.systemPrompt); + expect(second.prefix.tools).toBe(first.prefix.tools); + // The second request only appends to the first request's history. + expect(second.prefix.history.startsWith(first.prefix.history.slice(0, -1))).toBe(true); + + expect(first.prefix.systemPrompt).not.toContain("cache_note"); + expect(second.prefix.systemPrompt).not.toContain("cache_note"); + expect(second.prefix.history).not.toContain("cache_note"); + expect(second.volatile).toContain("cache_note"); + expect(second.volatile).toContain("Prompt caching needs a stable prefix."); + expect(second.volatile).not.toBe(first.volatile); + }); + + it("keeps the cached prefix byte-identical across a date flip", async () => { + vi.useFakeTimers({ toFake: ["Date"], now: new Date("2026-08-31T12:00:00Z") }); + const { harness, requests } = await createRecordingHarness({ + resourceLoader: { + ...createTestResourceLoader(), + getSystemPrompt: () => "You are a test assistant.", + }, + }); + harnesses.push(harness); + isolateGlobalHarnessState(harness); + + await harness.session.prompt("first"); + const firstDate = localDate(); + vi.setSystemTime(new Date("2026-09-02T12:00:00Z")); + const secondDate = localDate(); + harness.session.setActiveToolsByName(harness.session.getActiveToolNames()); + await harness.session.prompt("second"); + + expect(secondDate).not.toBe(firstDate); + expect(requests).toHaveLength(2); + const [first, second] = requests; + + expect(second.prefix.systemPrompt).toBe(first.prefix.systemPrompt); + expect(second.prefix.tools).toBe(first.prefix.tools); + expect(first.prefix.systemPrompt).not.toContain("Current date"); + expect(second.prefix.systemPrompt).not.toContain("Current date"); + expect(first.volatile).toContain(`Current date: ${firstDate}`); + expect(second.volatile).toContain(`Current date: ${secondDate}`); + }); + + it("serializes an unchanged tool set to identical bytes across registry refreshes", async () => { + const { harness, requests } = await createRecordingHarness(); + harnesses.push(harness); + isolateGlobalHarnessState(harness); + + await harness.session.prompt("first"); + const session = harness.session as unknown as { _refreshToolRegistry: () => void }; + session._refreshToolRegistry(); + session._refreshToolRegistry(); + await harness.session.prompt("second"); + + expect(requests).toHaveLength(2); + expect(requests[1].prefix.tools).toBe(requests[0].prefix.tools); + }); + + it("pins tool order to first activation so reordering cannot move the cache marker", async () => { + const { harness } = await createRecordingHarness(); + harnesses.push(harness); + isolateGlobalHarnessState(harness); + + const initialOrder = harness.session.getActiveToolNames(); + expect(initialOrder).toEqual(["alpha", "beta", "gamma"]); + + harness.session.setActiveToolsByName(["gamma", "beta", "alpha"]); + expect(harness.session.getActiveToolNames()).toEqual(["alpha", "beta", "gamma"]); + + // A tool that leaves and comes back returns to its original slot. + harness.session.setActiveToolsByName(["alpha", "gamma"]); + expect(harness.session.getActiveToolNames()).toEqual(["alpha", "gamma"]); + harness.session.setActiveToolsByName(["gamma", "alpha", "beta"]); + expect(harness.session.getActiveToolNames()).toEqual(["alpha", "beta", "gamma"]); + }); +}); diff --git a/packages/coding-agent/test/system-prompt.test.ts b/packages/coding-agent/test/system-prompt.test.ts index ca39619eb1..0d828aa838 100644 --- a/packages/coding-agent/test/system-prompt.test.ts +++ b/packages/coding-agent/test/system-prompt.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from "vitest"; import { buildRlmPrompt } from "../src/core/prompts/index.js"; import type { HarnessState } from "../src/core/refinement/index.js"; import type { Skill } from "../src/core/skills.js"; -import { buildSystemPrompt } from "../src/core/system-prompt.js"; +import { buildSystemPrompt, buildVolatileContext } from "../src/core/system-prompt.js"; import { createIpythonToolDefinition } from "../src/core/tools/ipython.js"; function skill(name: string): Skill { @@ -234,7 +234,7 @@ describe("buildSystemPrompt", () => { expect(shellPrompt).not.toContain("Generic MCP Connections"); }); - test("injects compact global harness context and refine guidance by default", () => { + test("keeps compact global harness context and refine guidance in the volatile block", () => { const harnessState: HarnessState = { schema: 1, entries: { @@ -322,45 +322,48 @@ describe("buildSystemPrompt", () => { ], }; - const prompt = buildSystemPrompt({ + const options = { selectedTools: ["ipython"], contextFiles: [], skills: [pythonSkill("refine"), pythonSkill("agent-message"), pythonSkill("agent-observe")], cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", harnessState, - }); - - expect(prompt).toContain("# Continual Harness State"); - expect(prompt).toContain("Local continual harness entries belong to this Prime Agent session"); - expect(prompt).toContain("The continual harness entries below are compact summaries, not full descriptions"); - expect(prompt).toContain("Use global continual harness refinement only for stable cross-session lessons"); - expect(prompt).toContain("When to call `await refine.run()`"); - expect(prompt).toContain("Call contract: read each installed Python skill's SKILL.md"); - expect(prompt).toContain("Continual harness skill entries are Python REPL skills"); - expect(prompt).toContain("Spawn a continual harness subagent spec by composing a concise task prompt"); - expect(prompt).toContain("handle = await rlm('sub-task')"); - expect(prompt).toContain("admission returns immediately"); - expect(prompt).toContain("never the child's answer"); - expect(prompt).toContain("receiver_role='parent'"); - expect(prompt).toContain("await rlm.list_subagents()"); - expect(prompt).toContain("receiver_role='child'"); - expect(prompt).not.toContain("asyncio.create_task(rlm('sub-task'))"); - expect(prompt).not.toContain("asyncio.gather(rlm('task1'), rlm('task2'))"); - expect(prompt).toContain("after a repeated failure"); - expect(prompt).toContain("a reusable tactic emerges"); - expect(prompt).toContain("a repeated delegation role should become a subagent spec"); - expect(prompt).toContain("a repeated procedure should become a skill"); - expect(prompt).toContain("a durable fact/preference should become a memory"); - expect(prompt).toContain("a narrow behavioral policy should become a prompt addendum"); - expect(prompt).toContain("validation shows a continual harness entry is wrong"); - expect(prompt).toContain("[global:focused_edits] Focused edits (policy, v1)"); - expect(prompt).toContain("[global:validation] Validation (repo/prime-agent, v2): Run `npm run check`"); - expect(prompt).toContain("[global:review_refinement] Review refinement (quality, v1)"); - expect(prompt).toContain("[global:refinement_reviewer] Refinement reviewer (review, v1)"); - expect(prompt).toContain("recent refinements: 1"); - expect(prompt).toContain("[refine_1] Observed validation miss: create memory:validation"); - expect(prompt.indexOf("# Continual Harness State")).toBeGreaterThan(prompt.indexOf("Conversation log:")); + }; + const prompt = buildSystemPrompt(options); + const volatile = buildVolatileContext(options) ?? ""; + + expect(prompt).toContain("Conversation log:"); + expect(prompt).not.toContain("# Continual Harness State"); + expect(volatile).toContain("# Continual Harness State"); + expect(volatile).toContain("Local continual harness entries belong to this Prime Agent session"); + expect(volatile).toContain("The continual harness entries below are compact summaries, not full descriptions"); + expect(volatile).toContain("Use global continual harness refinement only for stable cross-session lessons"); + expect(volatile).toContain("When to call `await refine.run()`"); + expect(volatile).toContain("Call contract: read each installed Python skill's SKILL.md"); + expect(volatile).toContain("Continual harness skill entries are Python REPL skills"); + expect(volatile).toContain("Spawn a continual harness subagent spec by composing a concise task prompt"); + expect(volatile).toContain("handle = await rlm('sub-task')"); + expect(volatile).toContain("admission returns immediately"); + expect(volatile).toContain("never the child's answer"); + expect(volatile).toContain("receiver_role='parent'"); + expect(volatile).toContain("await rlm.list_subagents()"); + expect(volatile).toContain("receiver_role='child'"); + expect(volatile).not.toContain("asyncio.create_task(rlm('sub-task'))"); + expect(volatile).not.toContain("asyncio.gather(rlm('task1'), rlm('task2'))"); + expect(volatile).toContain("after a repeated failure"); + expect(volatile).toContain("a reusable tactic emerges"); + expect(volatile).toContain("a repeated delegation role should become a subagent spec"); + expect(volatile).toContain("a repeated procedure should become a skill"); + expect(volatile).toContain("a durable fact/preference should become a memory"); + expect(volatile).toContain("a narrow behavioral policy should become a prompt addendum"); + expect(volatile).toContain("validation shows a continual harness entry is wrong"); + expect(volatile).toContain("[global:focused_edits] Focused edits (policy, v1)"); + expect(volatile).toContain("[global:validation] Validation (repo/prime-agent, v2): Run `npm run check`"); + expect(volatile).toContain("[global:review_refinement] Review refinement (quality, v1)"); + expect(volatile).toContain("[global:refinement_reviewer] Refinement reviewer (review, v1)"); + expect(volatile).toContain("recent refinements: 1"); + expect(volatile).toContain("[refine_1] Observed validation miss: create memory:validation"); }); test("keeps injected harness context compact", () => { @@ -393,18 +396,19 @@ describe("buildSystemPrompt", () => { refinements: [], }; - const prompt = buildSystemPrompt({ - selectedTools: ["ipython"], - contextFiles: [], - skills: [], - cwd: "/repo", - harnessState, - }); + const volatile = + buildVolatileContext({ + selectedTools: ["ipython"], + contextFiles: [], + skills: [], + cwd: "/repo", + harnessState, + }) ?? ""; - expect(prompt).toContain("memory: 8"); - expect(prompt).toContain("- +2 more memory entries"); - expect(prompt).toContain(`${"x".repeat(177)}...`); - expect(prompt).not.toContain(longContent); + expect(volatile).toContain("memory: 8"); + expect(volatile).toContain("- +2 more memory entries"); + expect(volatile).toContain(`${"x".repeat(177)}...`); + expect(volatile).not.toContain(longContent); }); test("uses the model-agnostic rlm harness prompt", () => { @@ -459,25 +463,29 @@ describe("buildSystemPrompt", () => { }, refinements: [], }; - const prompt = buildSystemPrompt({ + const options = { selectedTools: ["bash"], contextFiles: [], skills: [], cwd: "/repo", messagesPath: "/repo/.pi/sessions/session.jsonl", harnessState, - }); + }; + const prompt = buildSystemPrompt(options); + const volatile = buildVolatileContext(options) ?? ""; expect(prompt).toContain("You are a general purpose agent that uses code to solve tasks."); - expect(prompt).toContain("# Continual Harness State"); - expect(prompt).toContain("Call contract: use installed skills as shell commands"); - expect(prompt).toContain("subagent: 1"); - expect(prompt).not.toContain("persistent Python REPL"); - expect(prompt).not.toContain("Default to non-blocking subagents"); - expect(prompt).not.toContain("agent_observe.list_agents"); - expect(prompt).not.toContain("asyncio.create_task"); - expect(prompt).not.toContain("await "); - expect(prompt).not.toContain("await refine.run()"); + expect(volatile).toContain("# Continual Harness State"); + expect(volatile).toContain("Call contract: use installed skills as shell commands"); + expect(volatile).toContain("subagent: 1"); + for (const text of [prompt, volatile]) { + expect(text).not.toContain("persistent Python REPL"); + expect(text).not.toContain("Default to non-blocking subagents"); + expect(text).not.toContain("agent_observe.list_agents"); + expect(text).not.toContain("asyncio.create_task"); + expect(text).not.toContain("await "); + expect(text).not.toContain("await refine.run()"); + } }); test("omits shell guidance from harness state when shell is inactive", () => { @@ -506,22 +514,23 @@ describe("buildSystemPrompt", () => { }, refinements: [], }; - const prompt = buildSystemPrompt({ - selectedTools: ["edit"], - contextFiles: [], - skills: [], - cwd: "/repo", - messagesPath: "/repo/.pi/sessions/session.jsonl", - harnessState, - }); - - expect(prompt).toContain("# Continual Harness State"); - expect(prompt).toContain("without the Python REPL or shell access"); - expect(prompt).not.toContain("use installed skills as shell commands"); - expect(prompt).not.toContain(" ..."); - expect(prompt).not.toContain("asyncio.create_task"); - expect(prompt).not.toContain("await "); - expect(prompt).not.toContain("await refine.run()"); + const volatile = + buildVolatileContext({ + selectedTools: ["edit"], + contextFiles: [], + skills: [], + cwd: "/repo", + messagesPath: "/repo/.pi/sessions/session.jsonl", + harnessState, + }) ?? ""; + + expect(volatile).toContain("# Continual Harness State"); + expect(volatile).toContain("without the Python REPL or shell access"); + expect(volatile).not.toContain("use installed skills as shell commands"); + expect(volatile).not.toContain(" ..."); + expect(volatile).not.toContain("asyncio.create_task"); + expect(volatile).not.toContain("await "); + expect(volatile).not.toContain("await refine.run()"); }); test("custom prompt override bypasses the rlm harness body", () => { @@ -551,7 +560,7 @@ describe("buildSystemPrompt", () => { refinements: [], }; - const prompt = buildSystemPrompt({ + const options = { customPrompt: "custom body", selectedTools: ["ipython"], appendSystemPrompt: "custom append", @@ -559,18 +568,20 @@ describe("buildSystemPrompt", () => { skills: [], cwd: "/repo", harnessState, - }); + }; + const prompt = buildSystemPrompt(options); + const volatile = buildVolatileContext(options) ?? ""; expect(prompt).toContain("custom body"); - expect(prompt).toContain("# Continual Harness State"); - expect(prompt).toContain("[global:custom_memory] Custom memory (custom, v1)"); + expect(prompt).not.toContain("# Continual Harness State"); + expect(prompt).not.toContain("Current date:"); + expect(volatile).toContain("# Continual Harness State"); + expect(volatile).toContain("[global:custom_memory] Custom memory (custom, v1)"); + expect(volatile).toMatch(/^Current date: \d{4}-\d{2}-\d{2}$/m); + expect(volatile.indexOf("Current date:")).toBeLessThan(volatile.indexOf("# Continual Harness State")); expect(prompt).not.toContain("# IPython Kernel Guidance"); expect(prompt).not.toContain("You are a general purpose agent that uses code to solve tasks."); - expect(prompt.indexOf("Current working directory: /repo")).toBeLessThan( - prompt.indexOf("# Continual Harness State"), - ); expect(prompt.indexOf("Current working directory: /repo")).toBeLessThan(prompt.indexOf("custom append")); - expect(prompt.indexOf("# Continual Harness State")).toBeLessThan(prompt.indexOf("custom append")); }); test("adds child reply doctrine to custom prompts when messaging is available", () => { From 227c80e270c7b58c568953a1aa7faaf247cf1860 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Mon, 31 Aug 2026 01:27:18 -0600 Subject: [PATCH 2/3] docs(pylon): record the stable-prompt-cache-prefix pull request --- .pylon/features.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylon/features.yaml b/.pylon/features.yaml index 03f37fe454..b633e4aee2 100644 --- a/.pylon/features.yaml +++ b/.pylon/features.yaml @@ -325,7 +325,7 @@ decisions: pylon_refs: - https://github.com/pylon-code/prime-agent/issues/22 - https://github.com/pylon-code/prime-agent/issues/26 - - https://github.com/pylon-code/prime-agent/pull/32 + - https://github.com/pylon-code/prime-agent/pull/36 upstream_refs: - https://github.com/PrimeIntellect-ai/prime-agent/commit/4b9e6006fad553dcca5aa69b23a039088064589f - https://github.com/PrimeIntellect-ai/prime-agent/commit/f81acc6679ab214f1c40821588d9e5e97c47bdb1 From 4d762d0fdf16872b105187fcd9ebbbd9a618bc90 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Mon, 31 Aug 2026 01:40:52 -0600 Subject: [PATCH 3/3] fix(ai): declare append-only history for session-cached backends Trailing volatile placement only pays off for backends that cache by request prefix. Meridian's proxy matches the incoming message array against the history it already holds, and its lineage boundary tolerates a last-message append only for new tool_result blocks, so a payload-only volatile text block classified as modified-history and forced a full replay every turn. Model.appendOnlyHistory now declares that constraint. When set, the API registry routes volatileContext into the system prompt and adds no payload-only message block, so the request history stays byte-identical to the persisted conversation. Default placement is unchanged. The flag is settable from registerProvider model entries, models.json model definitions, and modelOverrides. refs #26 --- .pylon/features.yaml | 6 +- .pylon/upstream-review.md | 4 +- .../ai/.changes/stable-prompt-cache-prefix.md | 1 + packages/ai/README.md | 10 ++++ packages/ai/src/api-registry.ts | 20 ++++++- packages/ai/src/types.ts | 11 ++++ packages/ai/src/utils/volatile-context.ts | 18 ++++++ .../volatile-context-cache-prefix.test.ts | 59 +++++++++++++++++++ .../.changes/stable-prompt-cache-prefix.md | 1 + packages/coding-agent/docs/extensions.md | 28 +++++++++ .../coding-agent/src/core/model-registry.ts | 7 +++ .../26-stable-cache-prefix.test.ts | 49 +++++++++++++++ 12 files changed, 207 insertions(+), 7 deletions(-) diff --git a/.pylon/features.yaml b/.pylon/features.yaml index b633e4aee2..5bbd1ba632 100644 --- a/.pylon/features.yaml +++ b/.pylon/features.yaml @@ -331,7 +331,7 @@ decisions: - https://github.com/PrimeIntellect-ai/prime-agent/commit/f81acc6679ab214f1c40821588d9e5e97c47bdb1 - https://github.com/PrimeIntellect-ai/prime-agent/tree/c382f09856d4a8c8d2b765179657047d58691f25 fork_change: stable-prompt-cache-prefix-v1 - upstream_support: Prime through c382f09856d4 reduced the system-prompt date to a day stamp for cache stability, but it still serializes mutable continual-harness state and that date into the cached system prompt, has no provider-neutral volatile-content channel, and does not pin active tool order behind the `cache_control` marker on the last tool definition. + upstream_support: Prime through c382f09856d4 reduced the system-prompt date to a day stamp for cache stability, but it still serializes mutable continual-harness state and that date into the cached system prompt, has no provider-neutral volatile-content channel, no per-model placement contract for session-cached proxy backends, and does not pin active tool order behind the `cache_control` marker on the last tool definition. revisit_when: - - Prime upstream provides an equivalent provider-neutral channel that keeps mutable state and clock-derived content out of the cached tools/system/history prefix. - - Pylon can drop the fork behavior without regressing measured cache-hit rates through a Claude-Max proxy. + - Prime upstream provides an equivalent provider-neutral channel that keeps mutable state and clock-derived content out of the cached tools/system/history prefix, including a declared constraint for append-only-history backends. + - Pylon can drop the fork behavior without regressing measured cache-hit rates through a Claude-Max proxy or Meridian lineage continuation rates. diff --git a/.pylon/upstream-review.md b/.pylon/upstream-review.md index b91cffc759..ed4b70478d 100644 --- a/.pylon/upstream-review.md +++ b/.pylon/upstream-review.md @@ -157,6 +157,8 @@ This ledger records Prime upstream evidence and the decision taken for each over - Design: `Context.volatileContext` carries continual-harness state and the current date. The Anthropic provider appends it as the final content block after the `cache_control` breakpoint on the last history block, so the cached tools -> system -> history prefix is unchanged by a harness write or a date flip. The OpenAI-completions provider appends it after its Anthropic-style markers and after the automatically cached prefix of providers without markers. The API registry folds the content into the end of the message list for every other provider, so no provider can silently drop it. `AgentSession` pins each tool to the slot it first occupied, so a registry refresh or an extension reload serializes an unchanged tool set to identical bytes. - Behavior preservation: the harness-state section and the `Current date` line are byte-identical to what the system prompt previously inlined, including the empty-state rendering and the custom-prompt-only scope of the date. Only their position changed. The subagent guidance still precedes the harness-state roster in the assembled request. - Compatibility classification: **backward-compatible and additive**. `Context.volatileContext`, `AgentState.volatileContext`, `AgentLoopConfig.getVolatileContext`, and `ApiProvider.handlesVolatileContext` are all optional; a caller that sets none behaves exactly as before. No daemon command, event, protocol version, or schema revision changes. -- Validation: 10 new `packages/ai` payload tests assert a byte-identical cached prefix across volatile changes for both the Anthropic and OpenAI-completions payload builders, correct marker placement, the trailing-user-message case, blank-input handling, and the registry fallback. 4 new `packages/coding-agent/test/suite/regressions/26-stable-cache-prefix.test.ts` cases assert an unchanged prefix across a mid-session harness memory write, an unchanged prefix across a mocked two-day clock advance, identical tool bytes across repeated registry refreshes, and first-activation tool-order pinning across reordering and removal. `test/system-prompt.test.ts` (25), the compaction, compact-skill, refine, runtime, queue, and prompt suites (357), `packages/agent` (60), and 13 focused `packages/ai` provider files (93 passed, 9 gated skips) are green. `npm run check` passes. `test/extensions-runner.test.ts` fails 21 cases identically on unmodified `origin/pylon`, so it is pre-existing and unrelated. +- Validation: 15 `packages/ai` payload tests assert a byte-identical cached prefix across volatile changes for both the Anthropic and OpenAI-completions payload builders, correct marker placement, the trailing-user-message case, blank-input handling, and the registry fallback. 5 `packages/coding-agent/test/suite/regressions/26-stable-cache-prefix.test.ts` cases assert an unchanged prefix across a mid-session harness memory write, an unchanged prefix across a mocked two-day clock advance, identical tool bytes across repeated registry refreshes, first-activation tool-order pinning across reordering and removal, and — for an `appendOnlyHistory` model registered through `registerProvider` — an append-only history with the volatile content in the system prompt. `test/suite` (79 files, 960), `test/system-prompt.test.ts` (25), `test/model-registry.test.ts`, `test/agent-session-recursion.test.ts` (112), `packages/agent` (60), and 13 focused `packages/ai` provider files (93 passed, 9 gated skips) are green. `npm run check` passes. `test/extensions-runner.test.ts` fails 21 cases identically on unmodified `origin/pylon`, so it is pre-existing and unrelated. +- Review correction — two placements, declared per model. The first candidate always used trailing placement, which regressed the Meridian proxy path from "cold cache when harness state or the date changes" to a full fresh replay every turn. Meridian's lineage matcher (`meridian/src/proxy/session/lineage.ts`) resumes an SDK session only when the incoming history matches the stored history; its boundary branch at `cached.messageCount - 1` tolerates a last-message block append only when every stored block hash is preserved and every appended block is a new `tool_result` (`hasOnlyNewToolResults`, deliberately narrow per Meridian #689/#692). A payload-only volatile text block fails that on the turn it appears, and on the next turn the stored boundary message holds more blocks than the incoming replay because the block was never persisted, so `incomingBlockHashes.length > storedBlocks.length` fails too. Classification falls through to `diverged: modified-history` and forces a full-history replay on the exact deployment #22 targets. +- The fix is a declared per-model constraint, not a workaround. `Model.appendOnlyHistory` marks a backend that caches by session rather than by request prefix and therefore requires the request message array to stay byte-identical to the persisted history. When set, the API registry routes `volatileContext` into the system prompt and adds no payload-only message block; the backend's own session cache absorbs the system-prompt change. Default stays trailing placement, which is correct for direct prefix-cached APIs. The flag is settable from extension `registerProvider` model entries and from `models.json` model definitions and `modelOverrides`, and it is documented for extension authors in `packages/coding-agent/docs/extensions.md`. - Deferred: the optional debug-level prefix-drift check from issue #26 is not implemented. The regression tests assert the invariant directly, and a cross-turn byte comparison would need new mutable state inside otherwise pure prompt assembly. - Revisit when Prime upstream offers an equivalent provider-neutral channel that keeps mutable state and clock-derived content out of the cached prefix, and Pylon can drop the fork behavior without regressing measured cache-hit rates through a Claude-Max proxy. diff --git a/packages/ai/.changes/stable-prompt-cache-prefix.md b/packages/ai/.changes/stable-prompt-cache-prefix.md index a507ceae4c..3f294fc51e 100644 --- a/packages/ai/.changes/stable-prompt-cache-prefix.md +++ b/packages/ai/.changes/stable-prompt-cache-prefix.md @@ -1 +1,2 @@ - Added `Context.volatileContext` for content that must stay out of every cached prompt prefix; Anthropic and OpenAI-completions place it after their cache breakpoints and other providers receive it appended to the message list ([#26](https://github.com/pylon-code/prime-agent/issues/26)). +- Added the `Model.appendOnlyHistory` flag for session-cached backends, which routes volatile content into the system prompt so the request message array stays byte-identical to the persisted history ([#26](https://github.com/pylon-code/prime-agent/issues/26)). diff --git a/packages/ai/README.md b/packages/ai/README.md index 329b63edb3..90d4819bc6 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -997,6 +997,16 @@ The model still sees it, but never inside a cached region: - Anthropic and OpenAI-completions append it after their `cache_control` breakpoints, so the cached tools, system prompt, and conversation history are unaffected when it changes. - Every other provider receives it appended to the end of `Context.messages`, which is after the automatically cached prefix. Set `handlesVolatileContext: true` when registering a custom provider that positions the content itself. +### Session-cached backends + +Trailing placement assumes the backend caches by request prefix. A proxy in front of an agent SDK does not: it matches the incoming message array against the history it already holds and resumes that session, so a payload-only trailing block reads as a modified history and forces a full replay. + +Set `appendOnlyHistory: true` on those models. The volatile content then goes into the system prompt and the message array stays byte-identical to the caller's history: + +```typescript +const model = { ...getModel('anthropic', 'claude-opus-4-6'), baseUrl: 'https://my-sdk-proxy.example.com', appendOnlyHistory: true }; +``` + ## Context Serialization The `Context` object can be easily serialized and deserialized using standard JSON methods, making it simple to persist conversations, implement chat history, or transfer contexts between services: diff --git a/packages/ai/src/api-registry.ts b/packages/ai/src/api-registry.ts index 602cad7cbb..19cdaed2f4 100644 --- a/packages/ai/src/api-registry.ts +++ b/packages/ai/src/api-registry.ts @@ -7,7 +7,7 @@ import type { StreamFunction, StreamOptions, } from "./types.js"; -import { foldVolatileContext } from "./utils/volatile-context.js"; +import { foldVolatileContext, foldVolatileContextIntoSystemPrompt } from "./utils/volatile-context.js"; export type ApiStreamFunction = ( model: Model, @@ -46,6 +46,20 @@ type RegisteredApiProvider = { const apiProviderRegistry = new Map(); +/** + * Decide where `Context.volatileContext` goes before the provider sees it. + * + * Prefix-cached backends want it after the last cache breakpoint, which the + * provider places. Session-cached `appendOnlyHistory` backends need the message + * array untouched, so it goes into the system prompt instead. + */ +function resolveVolatilePlacement(model: Model, context: Context, handlesVolatileContext: boolean): Context { + if (model.appendOnlyHistory) { + return foldVolatileContextIntoSystemPrompt(context); + } + return handlesVolatileContext ? context : foldVolatileContext(context); +} + function wrapStream( api: TApi, stream: StreamFunction, @@ -55,7 +69,7 @@ function wrapStream( if (model.api !== api) { throw new Error(`Mismatched api: ${model.api} expected ${api}`); } - const resolved = handlesVolatileContext ? context : foldVolatileContext(context); + const resolved = resolveVolatilePlacement(model, context, handlesVolatileContext); return stream(model as Model, resolved, options as TOptions); }; } @@ -69,7 +83,7 @@ function wrapStreamSimple( if (model.api !== api) { throw new Error(`Mismatched api: ${model.api} expected ${api}`); } - const resolved = handlesVolatileContext ? context : foldVolatileContext(context); + const resolved = resolveVolatilePlacement(model, context, handlesVolatileContext); return streamSimple(model as Model, resolved, options); }; } diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 95da0fda30..3bcad49d95 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -462,6 +462,17 @@ export interface Model { maxTokens: number; /** Flagship model surfaced above non-featured models of the same provider in pickers. */ featured?: boolean; + /** + * Set when the backend keeps its own session cache keyed on the request message + * array, so that array must stay byte-identical to the conversation history the + * client persisted — a proxy in front of an agent SDK, for example. + * + * `Context.volatileContext` then goes into the system prompt instead of the end + * of the message list, because a payload-only message block breaks the backend's + * lineage matching and forces a full-history replay on every request. The + * backend's own session cache absorbs the system-prompt change instead. + */ + appendOnlyHistory?: boolean; headers?: Record; /** Compatibility overrides for OpenAI-compatible APIs. If not set, auto-detected from baseUrl. */ compat?: TApi extends "openai-completions" diff --git a/packages/ai/src/utils/volatile-context.ts b/packages/ai/src/utils/volatile-context.ts index 59254a5104..58f513a769 100644 --- a/packages/ai/src/utils/volatile-context.ts +++ b/packages/ai/src/utils/volatile-context.ts @@ -27,3 +27,21 @@ export function foldVolatileContext(context: Context): Context { const messages: Message[] = [...context.messages, trailing]; return { ...context, messages, volatileContext: undefined }; } + +/** + * Move volatile content to the end of the system prompt, leaving the message + * list byte-identical to the caller's history. + * + * Used for `Model.appendOnlyHistory` backends. They cache by session rather than + * by prefix, so a system-prompt change costs one re-send while a payload-only + * message block would invalidate their whole history lineage. + */ +export function foldVolatileContextIntoSystemPrompt(context: Context): Context { + const text = resolveVolatileContext(context); + if (!text) { + return context.volatileContext === undefined ? context : { ...context, volatileContext: undefined }; + } + + const systemPrompt = context.systemPrompt ? `${context.systemPrompt}\n\n${text}` : text; + return { ...context, systemPrompt, volatileContext: undefined }; +} diff --git a/packages/ai/test/volatile-context-cache-prefix.test.ts b/packages/ai/test/volatile-context-cache-prefix.test.ts index f6f0cb096b..54b5c1ef06 100644 --- a/packages/ai/test/volatile-context-cache-prefix.test.ts +++ b/packages/ai/test/volatile-context-cache-prefix.test.ts @@ -263,6 +263,35 @@ describe("Anthropic volatile context placement", () => { const payload = await captureAnthropicPayload(" \n "); expect(flattenParts(payload.messages[payload.messages.length - 1])).toHaveLength(1); }); + + it("sends volatile context in the system prompt for an appendOnlyHistory model", async () => { + const model: Model<"anthropic-messages"> = { + ...getModel("anthropic", "claude-sonnet-4-5"), + baseUrl: "http://127.0.0.1:9", + appendOnlyHistory: true, + }; + const source = baseContext("# Continual Harness State\n\nmemory: 1"); + let captured: CapturedPayload | undefined; + // Routed through the registry, which owns the placement decision. + await streamSimple(model, source, { + apiKey: "fake-key", + maxRetries: 0, + onPayload: (payload) => { + captured = payload as CapturedPayload; + return payload; + }, + }).result(); + + if (!captured) throw new Error("Expected the Anthropic payload to be captured"); + expect(JSON.stringify(captured.system)).toContain("Continual Harness State"); + + // No payload-only block: the message array still matches the caller's history. + const parts = flattenParts(captured.messages[captured.messages.length - 1]); + expect(parts).toHaveLength(1); + expect(parts[0]?.text).toBe("second"); + expect(parts[0]?.cache_control).toEqual({ type: "ephemeral" }); + expect(captured.messages).toHaveLength(source.messages.length); + }); }); describe("OpenAI-completions volatile context placement", () => { @@ -356,6 +385,36 @@ describe("volatile context handling at the provider registry", () => { expect(contexts[0].messages).toHaveLength(baseContext().messages.length); }); + it("routes volatile context into the system prompt for appendOnlyHistory models", () => { + for (const handlesVolatileContext of [true, false]) { + const api = handlesVolatileContext ? "native-api" : "fallback-api"; + const { contexts, model } = recordingProvider(api, handlesVolatileContext); + const source = baseContext("# Continual Harness State\n\nmemory: 1"); + + streamSimple({ ...model, appendOnlyHistory: true }, source); + + expect(contexts).toHaveLength(1); + const received = contexts[0]; + expect(received.volatileContext).toBeUndefined(); + // A session-cached backend matches on the message array, so it must be + // byte-identical to the caller's persisted history. + expect(JSON.stringify(received.messages)).toBe(JSON.stringify(source.messages)); + expect(received.systemPrompt).toBe( + "You are a general purpose agent.\n\n# Continual Harness State\n\nmemory: 1", + ); + unregisterApiProviders(api); + } + }); + + it("uses the volatile content as the system prompt when an appendOnlyHistory model has none", () => { + const { contexts, model } = recordingProvider("native-api", true); + + streamSimple({ ...model, appendOnlyHistory: true }, { messages: [], volatileContext: "memory: 1" }); + + expect(contexts[0].systemPrompt).toBe("memory: 1"); + expect(contexts[0].messages).toHaveLength(0); + }); + it("keeps volatile content out of the faux provider's simulated cache prefix", async () => { const faux = registerFauxProvider(); try { diff --git a/packages/coding-agent/.changes/stable-prompt-cache-prefix.md b/packages/coding-agent/.changes/stable-prompt-cache-prefix.md index e1deb985ca..1ad6b63fe6 100644 --- a/packages/coding-agent/.changes/stable-prompt-cache-prefix.md +++ b/packages/coding-agent/.changes/stable-prompt-cache-prefix.md @@ -1,2 +1,3 @@ - Fixed prompt-cache churn by sending continual harness state and the current date outside the cached prompt prefix, so a harness refinement or a midnight date flip no longer re-caches a long-running session ([#26](https://github.com/pylon-code/prime-agent/issues/26)). - Pinned each tool to the slot it first occupied so a tool-registry refresh serializes an unchanged tool set to identical bytes ([#26](https://github.com/pylon-code/prime-agent/issues/26)). +- Added the `appendOnlyHistory` model flag so proxy-backed providers registered by extensions keep an append-only request history and receive volatile prompt content in the system prompt ([#26](https://github.com/pylon-code/prime-agent/issues/26)). diff --git a/packages/coding-agent/docs/extensions.md b/packages/coding-agent/docs/extensions.md index cb5a71d71d..26f433f1c3 100644 --- a/packages/coding-agent/docs/extensions.md +++ b/packages/coding-agent/docs/extensions.md @@ -1648,6 +1648,34 @@ pi.registerProvider("corporate-ai", { - `oauth` - OAuth provider config for `/login` support. When provided, the provider appears in the login menu. - `streamSimple` - Custom streaming implementation for non-standard APIs. +**Model definitions: `appendOnlyHistory`** + +Set `appendOnlyHistory: true` on a model when the backend keeps its own session cache keyed on the request message array, rather than caching by request prefix. Proxies in front of an agent SDK work this way: they match the incoming history against the history they already hold and resume that session, so they only tolerate an append. + +The flag controls where Prime Agent puts volatile prompt content — continual harness state and the current date. By default that content is appended after the message list so a change cannot invalidate a prefix cache. A session-cached backend reads the same append as a modified history and replays the whole conversation instead, so the flag routes the content into the system prompt and leaves the message array byte-identical to the persisted conversation. + +```typescript +pi.registerProvider("my-sdk-proxy", { + baseUrl: "https://proxy.example.com", + apiKey: "PROXY_API_KEY", + api: "anthropic-messages", + models: [ + { + id: "claude-opus-4-6", + name: "Claude Opus 4.6 (via SDK proxy)", + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 32000, + appendOnlyHistory: true + } + ] +}); +``` + +Leave it unset for direct provider APIs, including Anthropic and OpenAI-compatible endpoints that cache by prefix. `models.json` accepts the same field on a model definition or a `modelOverrides` entry. + See [custom-provider.md](custom-provider.md) for advanced topics: custom streaming APIs, OAuth details, model definition reference. ### pi.unregisterProvider(name) diff --git a/packages/coding-agent/src/core/model-registry.ts b/packages/coding-agent/src/core/model-registry.ts index 4ebf6c3b8d..a44ee05049 100644 --- a/packages/coding-agent/src/core/model-registry.ts +++ b/packages/coding-agent/src/core/model-registry.ts @@ -159,6 +159,7 @@ const ModelDefinitionSchema = Type.Object({ ), contextWindow: Type.Optional(Type.Number()), maxTokens: Type.Optional(Type.Number()), + appendOnlyHistory: Type.Optional(Type.Boolean()), headers: Type.Optional(Type.Record(Type.String(), Type.String())), compat: Type.Optional(ProviderCompatSchema), }); @@ -178,6 +179,7 @@ const ModelOverrideSchema = Type.Object({ ), contextWindow: Type.Optional(Type.Number()), maxTokens: Type.Optional(Type.Number()), + appendOnlyHistory: Type.Optional(Type.Boolean()), headers: Type.Optional(Type.Record(Type.String(), Type.String())), compat: Type.Optional(ProviderCompatSchema), }); @@ -333,6 +335,7 @@ function applyModelOverride(model: Model, override: ModelOverride): Model); @@ -1541,6 +1545,7 @@ export class ModelRegistry { cost: modelDef.cost, contextWindow: modelDef.contextWindow, maxTokens: modelDef.maxTokens, + appendOnlyHistory: modelDef.appendOnlyHistory, headers: undefined, compat: modelDef.compat, } as Model); @@ -1587,6 +1592,8 @@ export interface ProviderConfigInput { cost: { input: number; output: number; cacheRead: number; cacheWrite: number }; contextWindow: number; maxTokens: number; + /** Set for proxy or session-cached backends that require an append-only message history. */ + appendOnlyHistory?: boolean; headers?: Record; compat?: Model["compat"]; }>; diff --git a/packages/coding-agent/test/suite/regressions/26-stable-cache-prefix.test.ts b/packages/coding-agent/test/suite/regressions/26-stable-cache-prefix.test.ts index 33056af14a..a113f6e6a5 100644 --- a/packages/coding-agent/test/suite/regressions/26-stable-cache-prefix.test.ts +++ b/packages/coding-agent/test/suite/regressions/26-stable-cache-prefix.test.ts @@ -208,6 +208,55 @@ describe("issue 26: stable prompt-cache prefix", () => { expect(requests[1].prefix.tools).toBe(requests[0].prefix.tools); }); + it("keeps the history append-only and moves volatile content into the system prompt for an appendOnlyHistory model", async () => { + const { harness, requests } = await createRecordingHarness(); + harnesses.push(harness); + isolateGlobalHarnessState(harness); + + const fauxModel = harness.getModel(); + harness.session.modelRegistry.registerProvider("append-only-proxy", { + baseUrl: fauxModel.baseUrl, + apiKey: "faux-key", + api: harness.faux.api, + models: [ + { + id: "append-only-model", + name: "Append Only Proxy", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 16384, + appendOnlyHistory: true, + }, + ], + }); + const registered = harness.session.modelRegistry.getAll().find((model) => model.id === "append-only-model"); + // The flag has to survive registerProvider parsing to reach the provider. + expect(registered?.appendOnlyHistory).toBe(true); + await harness.session.setModel(registered!); + + await harness.session.prompt("first"); + seedLocalMemory(harness, "proxy_note", "Session-cached backends need an append-only history."); + harness.session.setActiveToolsByName(harness.session.getActiveToolNames()); + await harness.session.prompt("second"); + + expect(requests).toHaveLength(2); + const [first, second] = requests; + + // No payload-only block anywhere: each request's history only appends to the last. + expect(first.volatile).toBe(""); + expect(second.volatile).toBe(""); + expect(second.prefix.history.startsWith(first.prefix.history.slice(0, -1))).toBe(true); + expect(first.prefix.history).not.toContain("Continual Harness State"); + expect(second.prefix.history).not.toContain("Continual Harness State"); + + // The content still reaches the model, inside the system prompt. + expect(first.prefix.systemPrompt).toContain("# Continual Harness State"); + expect(second.prefix.systemPrompt).toContain("proxy_note"); + expect(second.prefix.systemPrompt).toContain("Session-cached backends need an append-only history."); + }); + it("pins tool order to first activation so reordering cannot move the cache marker", async () => { const { harness } = await createRecordingHarness(); harnesses.push(harness);