From e451dc6ffc5aa0042fe4781d9111393cec8879db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Fri, 26 Jun 2026 23:31:54 +0200 Subject: [PATCH 01/22] docs: design for self-hosted LLMs via custom Pi providers (LiteLLM) Adds the brainstormed design spec for registering OpenAI-compatible self-hosted endpoints (e.g. LiteLLM) as named Pi providers in warden.toml, covering all model lanes with env-based auth. Co-Authored-By: Claude Opus 4.8 --- ...6-warden-custom-provider-litellm-design.md | 219 ++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-26-warden-custom-provider-litellm-design.md diff --git a/docs/superpowers/specs/2026-06-26-warden-custom-provider-litellm-design.md b/docs/superpowers/specs/2026-06-26-warden-custom-provider-litellm-design.md new file mode 100644 index 00000000..1777fa79 --- /dev/null +++ b/docs/superpowers/specs/2026-06-26-warden-custom-provider-litellm-design.md @@ -0,0 +1,219 @@ +# Design: Self-hosted LLMs via custom Pi providers (LiteLLM) + +**Date:** 2026-06-26 +**Status:** Approved (pending spec review) +**Topic:** Custom OpenAI-compatible provider support for the Pi runtime + +## Problem + +Warden can only target the providers Pi ships built-in (Anthropic, OpenAI, +OpenRouter, Fireworks, etc.) or the Claude runtime. Users who run their own +models behind a self-hosted gateway such as [LiteLLM](https://docs.litellm.ai/) +have no first-class way to point Warden at a custom base URL. LiteLLM exposes an +OpenAI-compatible `/v1/chat/completions` surface (and an Anthropic-compatible +`/v1/messages` surface), so the capability exists at the SDK layer but is not +reachable from `warden.toml`. + +## Goal + +Let a user register a self-hosted, OpenAI-compatible endpoint as a named provider +in `warden.toml` and select its models with the existing `provider/model` +selector syntax. All model lanes (agent, auxiliary, synthesis) route through the +custom provider when their selector points at it. + +## Non-goals (YAGNI) + +- Per-skill or per-trigger provider definitions. Providers are global within a + config layer, consistent with how `runtime` already works. +- Formalizing `ANTHROPIC_BASE_URL` for the `claude` runtime. (It already works + via env passthrough; not part of this change.) +- Compatibility modes beyond OpenAI-completions. The schema reserves an `api` + field so `anthropic-messages` can be added later, but only + `openai-completions` is wired and tested now. +- Storing secrets in `warden.toml`. API keys come from the environment only. + +## Key facts (verified against installed packages) + +- `@earendil-works/pi-coding-agent@0.78.0` `ModelRegistry` exposes + `registerProvider(name, config: ProviderConfigInput)` where + `ProviderConfigInput` accepts `{ baseUrl, apiKey, api, headers, authHeader, + models[] }`. Each `models[]` entry requires + `{ id, name, reasoning, input, cost{input,output,cacheRead,cacheWrite}, + contextWindow, maxTokens }` and optional `{ api, baseUrl, headers, compat, + thinkingLevelMap }`. +- `@earendil-works/pi-ai@0.78.0` `Api` includes `"openai-completions"`. + `OpenAICompletionsCompat` is auto-detected from the base URL when `compat` is + omitted. +- `pi.ts` `runPiPrompt()` is the single choke point: both `runSkill` and the + auxiliary/synthesis `runStructured` paths call it. It builds a fresh + `ModelRegistry.create(authStorage)` per call, then `resolvePiModel()`. + Registering the custom provider here covers every lane. +- `AuthStorage.setRuntimeApiKey(provider, apiKey)` already sets the legacy + Anthropic key today (`createAuthStorage`). The same mechanism sets a custom + provider key. +- `bridgeWardenProviderApiKeyEnv()` already mirrors `WARDEN__API_KEY` to + `_API_KEY`. +- The agent path (`analyze.ts`, `verify.ts`) already builds provider options via + `getRuntimeProviderOptions(runtimeName, {...})` and threads a `providerOptions` + field on `SkillRunRequest`. `AuxiliaryRunRequest`/`SynthesisRunRequest` do + **not** yet carry `providerOptions`; this change adds it. + +## Configuration surface + +New `[defaults.providers.]` map in `warden.toml`: + +```toml +[defaults] +runtime = "pi" + +[defaults.providers.litellm] +baseUrl = "http://localhost:4000/v1" # required; OpenAI-compatible base URL +api = "openai-completions" # optional; default "openai-completions" +# headers = { "X-Tenant" = "team-a" } # optional custom headers +# apiKeyEnv = "WARDEN_LITELLM_API_KEY" # optional; override the default env lookup + +[[defaults.providers.litellm.models]] +id = "my-model" # required; the model name LiteLLM exposes +# contextWindow = 128000 # optional; default 128000 +# maxTokens = 8192 # optional; default 8192 +# reasoning = false # optional; default false +# cost = { input = 0, output = 0, cacheRead = 0, cacheWrite = 0 } # default zeros + +[defaults.agent] +model = "litellm/my-model" +``` + +Selector behavior is unchanged: split at the first `/`, provider before, model id +after. `litellm/my-model` resolves to the registered `litellm` provider. + +### Model field defaults + +Only `id` is required per model. Warden fills the rest before calling +`registerProvider`: + +| Field | Default | +| --- | --- | +| `name` | same as `id` | +| `reasoning` | `false` | +| `input` | `["text"]` | +| `contextWindow` | `128000` | +| `maxTokens` | `8192` | +| `cost` | `{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }` | +| `api` | provider-level `api` (default `openai-completions`) | + +Cost defaults of zero mean self-hosted runs report `$0`; acceptable for the +common internal-gateway case. + +## Authentication + +No secret lives in `warden.toml`. The Bearer key is resolved from the +environment at runtime: + +1. If `apiKeyEnv` is set, read that variable. +2. Otherwise try `WARDEN__API_KEY`, then `_API_KEY` + (e.g. `WARDEN_LITELLM_API_KEY`, then `LITELLM_API_KEY`). +3. If none is set and the endpoint is non-loopback, fail preflight with a clear + message naming the expected env var. Loopback URLs (localhost/127.0.0.1) are + allowed to run unauthenticated. + +The resolved key is handed to Pi via `registerProvider({ apiKey })` and/or +`AuthStorage.setRuntimeApiKey(name, key)`. It never touches disk. + +## Architecture / plumbing + +### 1. Schema (`config/schema.ts`) + +- `ProviderModelSchema`: `{ id: string (min 1), name?: string, + reasoning?: boolean, input?: ("text"|"image")[], contextWindow?: positive int, + maxTokens?: positive int, cost?: { input, output, cacheRead, cacheWrite all + >= 0 } }`. +- `ProviderConfigSchema`: `{ baseUrl: z.string().url(), + api?: z.enum(["openai-completions"]) (default "openai-completions"), + headers?: z.record(z.string()), apiKeyEnv?: z.string(), + models: z.array(ProviderModelSchema).min(1) }` (`.strict()`). +- `ProvidersConfigSchema`: `z.record(z.string(), ProviderConfigSchema)`. +- Add `providers: ProvidersConfigSchema.optional()` to `DefaultsSchema`. + +### 2. Provider options builder (`runtimes/index.ts`) + +- Extend `RuntimeProviderOptionsInput` with an optional `providers` field + (the validated `ProvidersConfig`). +- In `getRuntimeProviderOptions`, when `name === 'pi'` and `providers` is + present, return a `PiProviderOptions` object: the normalized provider list with + model defaults applied and the env-resolved API key attached per provider. + Key resolution (env lookup) happens here so the Pi adapter receives ready + values; absent keys are left undefined for the adapter/preflight to handle. +- `claude` branch unchanged. + +### 3. Pi adapter (`runtimes/pi.ts`) + +- Add `providers?: PiProviderOptions` to `PiPromptOptions`. +- In `runPiPrompt`, after `ModelRegistry.create(authStorage)` and before + `resolvePiModel`, iterate `providers` and call + `modelRegistry.registerProvider(name, { baseUrl, api, headers, apiKey, + models })`. Set the runtime API key on `authStorage` when present. +- `runSkill` reads `providers` from `request.providerOptions`. +- `runStructured` (auxiliary/synthesis) reads `providers` from the request and + forwards into `runPiPrompt`. + +### 4. Request types (`runtimes/types.ts`) + +- Add `providerOptions?: unknown` to `AuxiliaryRunRequestBase` and + `SynthesisRunRequest` so auxiliary/synthesis calls can carry the same + provider options the agent path already passes. + +### 5. Threading (all lanes) + +Pass the `providers` config into `getRuntimeProviderOptions` at every Pi call +site, mirroring how `runtimeName`/`model` already flow: + +- Agent: `analyze.ts`, `verify.ts` (extend the existing + `getRuntimeProviderOptions` calls with `providers`). +- Auxiliary/synthesis: `extract.ts`, `output/dedup.ts`, + `action/fix-evaluation/judge.ts`, `sdk/json-output.ts`, and the skill-builder + paths (`outline.ts`, `agentic.ts`, `skill.ts`) — add `providerOptions: + getRuntimeProviderOptions(runtimeName, { providers })` to each runtime request. + +The `providers` value originates from the resolved `WardenConfig.defaults` and +is carried alongside the existing runner options object that already conveys +`runtime`/`model` to these call sites. + +### 6. Validation & errors + +- When resolving a model selector whose provider prefix is neither a built-in Pi + provider nor a key in `[defaults.providers]`, throw a clear error naming the + unknown provider and listing configured custom providers. Reuse the existing + invalid-selector error surface where practical. +- Preflight: if a configured provider requires a key (has `apiKeyEnv` or a + non-loopback `baseUrl`) and none resolves, fail before the first model call. + +## Testing + +- **Unit (`config/schema.test.ts`)**: accept a valid `[defaults.providers.*]` + block; reject a bad `baseUrl`, an unsupported `api`, and an empty `models` + array. +- **Unit (`runtimes/index.test.ts`)**: `getRuntimeProviderOptions('pi', + { providers })` applies model defaults and resolves the API key from + `apiKeyEnv`, then `WARDEN__API_KEY`, then `_API_KEY`; returns + undefined key when absent. +- **Integration (`runtimes/pi.test.ts`)**: with `ModelRegistry`/ + `registerProvider` stubbed, a request carrying a custom provider registers it + and resolves `litellm/my-model`. Mock the HTTP boundary; no live LiteLLM. +- **Regression**: a selector with an unknown provider prefix produces the + friendly error. + +## Documentation + +Extend `packages/docs/src/content/docs/config/models.mdx` with a "Self-hosted / +OpenAI-compatible providers (LiteLLM)" section covering the +`[defaults.providers.*]` block, the model-field defaults table, env-var auth, +and the Pi `models.json` passthrough as an alternative escape hatch. + +## Decisions captured + +- Runtime: Pi only. +- Config surface: first-class `warden.toml` + documented `models.json` + passthrough. +- Endpoint shape: OpenAI-compatible with Bearer key. +- Lane coverage: all lanes (agent + auxiliary + synthesis). +- Model fields: sensible defaults; only `id` required. From 5bc7bbc456306d183918184f65454adb6d4ec8f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Fri, 26 Jun 2026 23:44:17 +0200 Subject: [PATCH 02/22] docs: implementation plan for custom Pi providers (LiteLLM) Co-Authored-By: Claude Opus 4.8 --- ...26-06-26-warden-custom-provider-litellm.md | 979 ++++++++++++++++++ 1 file changed, 979 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-26-warden-custom-provider-litellm.md diff --git a/docs/superpowers/plans/2026-06-26-warden-custom-provider-litellm.md b/docs/superpowers/plans/2026-06-26-warden-custom-provider-litellm.md new file mode 100644 index 00000000..622f7067 --- /dev/null +++ b/docs/superpowers/plans/2026-06-26-warden-custom-provider-litellm.md @@ -0,0 +1,979 @@ +# Custom Pi Provider (LiteLLM / self-hosted) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let users register a self-hosted, OpenAI-compatible endpoint (e.g. LiteLLM) as a named provider in `warden.toml` and target its models with the existing `provider/model` selector, across all model lanes. + +**Architecture:** Add a `[defaults.providers.]` config block, normalize it (with sensible per-model defaults) into a Pi provider-options object whose API key is resolved from the environment, and register it on Pi's `ModelRegistry` via `registerProvider` inside `runPiPrompt` (the single choke point all Pi lanes flow through). Thread the raw config from the resolved trigger config to every runtime call site alongside the existing `runtime`/`auxiliaryModel` fields. + +**Tech Stack:** TypeScript (strict, ESM), Zod v4, Vitest, `@earendil-works/pi-coding-agent@0.78.0` (`ModelRegistry.registerProvider`), `@earendil-works/pi-ai@0.78.0` (`Api = "openai-completions"`). + +## Global Constraints + +- TypeScript strict mode; use `export type` for type-only exports (Bun compatibility). +- Zod for all runtime validation; new schemas live in `config/schema.ts`. +- No secrets in `warden.toml`. API keys resolve from env only. +- Co-locate tests (`foo.ts` → `foo.test.ts`). Prefer integration over unit; mock the HTTP/SDK boundary, never call a live endpoint. +- Verify with `pnpm lint && pnpm build && pnpm test` (run targeted vitest during tasks). +- Commit messages use the repo convention and end with `Co-Authored-By: Claude Opus 4.8 `. +- Only `openai-completions` is a valid `api` value in v1. The field exists so `anthropic-messages` can be added later. + +--- + +## File Structure + +- **Create** `packages/warden/src/sdk/runtimes/custom-provider.ts` — normalization, env key resolution, auth assertion. Pure functions, no I/O beyond reading a passed-in env object. +- **Create** `packages/warden/src/sdk/runtimes/custom-provider.test.ts` — unit tests for the above. +- **Modify** `packages/warden/src/config/schema.ts` — add provider schemas + `DefaultsSchema.providers`. +- **Modify** `packages/warden/src/config/schema.test.ts` (or `config/loader.test.ts` if no schema test exists) — schema accept/reject tests. +- **Modify** `packages/warden/src/sdk/runtimes/index.ts` — extend `getRuntimeProviderOptions` for `pi`. +- **Modify** `packages/warden/src/sdk/runtimes/types.ts` — add `providerOptions` to auxiliary/synthesis requests. +- **Modify** `packages/warden/src/sdk/runtimes/pi.ts` — register custom providers in `runPiPrompt`; read `providerOptions` in `runSkill`/`runStructured`. +- **Modify** `packages/warden/src/sdk/runtimes/pi.test.ts` — registration integration test. +- **Modify** `packages/warden/src/sdk/types.ts` — `SkillRunnerOptions.providers`. +- **Modify** `packages/warden/src/sdk/analyze.ts`, `verify.ts`, `extract.ts` — thread `providers` into agent + extraction runtime calls. +- **Modify** `packages/warden/src/output/dedup.ts`, `action/fix-evaluation/judge.ts`, `sdk/json-output.ts` — thread `providers` into auxiliary/synthesis calls. +- **Modify** `packages/warden/src/config/loader.ts` — `ResolvedTrigger.providers` + assignment. +- **Modify** runner entry points: `action/triggers/executor.ts`, `action/workflow/schedule.ts`, `action/workflow/pr-workflow.ts`, `action/review/poster.ts`, `cli/main.ts` — copy `providers` alongside `auxiliaryModel`. +- **Modify** `packages/docs/src/content/docs/config/models.mdx` — docs. + +--- + +## Task 1: Provider config schema + +**Files:** +- Modify: `packages/warden/src/config/schema.ts` (add near `DefaultsSchema`, before line 220) +- Test: `packages/warden/src/config/loader.test.ts` (add a `describe` block; this file already imports the schema/loader) + +**Interfaces:** +- Produces: `ProviderModelConfigSchema`, `ProviderConfigSchema`, `ProvidersConfigSchema`; types `ProviderModelConfig`, `ProviderConfig`, `ProvidersConfig`; new optional `DefaultsSchema.providers` field of type `ProvidersConfig`. + +- [ ] **Step 1: Write the failing test** + +Add to `packages/warden/src/config/loader.test.ts` (top-level, near other `describe`s). It parses TOML through the existing loader the same way other tests in this file do — check the file head for the existing `parseConfig`/`loadConfig` import and reuse it. If the file exposes `WardenConfigSchema` parsing helpers, prefer those. The assertions: + +```ts +import { WardenConfigSchema } from './schema.js'; + +describe('providers config', () => { + const base = { version: 1 as const, skills: [] }; + + it('accepts a valid custom provider', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { + providers: { + litellm: { + baseUrl: 'http://localhost:4000/v1', + models: [{ id: 'my-model' }], + }, + }, + }, + }); + expect(result.success).toBe(true); + if (result.success) { + const p = result.data.defaults?.providers?.['litellm']; + expect(p?.api).toBe('openai-completions'); // default applied + expect(p?.models[0]?.id).toBe('my-model'); + } + }); + + it('rejects a non-URL baseUrl', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { litellm: { baseUrl: 'not a url', models: [{ id: 'm' }] } } }, + }); + expect(result.success).toBe(false); + }); + + it('rejects an unsupported api value', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { x: { baseUrl: 'http://h/v1', api: 'cohere', models: [{ id: 'm' }] } } }, + }); + expect(result.success).toBe(false); + }); + + it('rejects an empty models array', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { x: { baseUrl: 'http://h/v1', models: [] } } }, + }); + expect(result.success).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- config/loader.test.ts -t "providers config"` +Expected: FAIL (TOML parses but `providers` is stripped/unknown, so `api` default assertion fails; reject cases pass-through as success). + +- [ ] **Step 3: Add the schemas** + +In `packages/warden/src/config/schema.ts`, insert before `DefaultsSchema` (currently line 220): + +```ts +// Per-model definition for a custom provider. Only `id` is required; Warden +// fills the remaining fields Pi requires with sensible defaults. +export const ProviderModelConfigSchema = z.object({ + /** Model id as exposed by the endpoint (e.g. the LiteLLM model name). */ + id: z.string().min(1), + /** Display name. Defaults to `id`. */ + name: z.string().min(1).optional(), + /** Whether the model supports reasoning/thinking. Default: false. */ + reasoning: z.boolean().optional(), + /** Accepted input modalities. Default: ["text"]. */ + input: z.array(z.enum(['text', 'image'])).min(1).optional(), + /** Context window in tokens. Default: 128000. */ + contextWindow: z.number().int().positive().optional(), + /** Max output tokens. Default: 8192. */ + maxTokens: z.number().int().positive().optional(), + /** Per-token cost (USD per token). Default: all zeros. */ + cost: z.object({ + input: z.number().nonnegative(), + output: z.number().nonnegative(), + cacheRead: z.number().nonnegative(), + cacheWrite: z.number().nonnegative(), + }).optional(), +}).strict(); +export type ProviderModelConfig = z.infer; + +// A custom, self-hosted or gateway provider (e.g. LiteLLM). OpenAI-compatible only in v1. +export const ProviderConfigSchema = z.object({ + /** OpenAI-compatible base URL, typically ending in /v1. */ + baseUrl: z.string().url(), + /** Wire protocol. Only "openai-completions" is supported today. */ + api: z.enum(['openai-completions']).default('openai-completions'), + /** Extra HTTP headers sent on every request. */ + headers: z.record(z.string(), z.string()).optional(), + /** Env var holding the API key. Defaults to WARDEN__API_KEY then _API_KEY. */ + apiKeyEnv: z.string().min(1).optional(), + /** Models this provider exposes. At least one is required. */ + models: z.array(ProviderModelConfigSchema).min(1), +}).strict(); +export type ProviderConfig = z.infer; + +// Map of provider name -> provider config. The name becomes the model-selector prefix. +export const ProvidersConfigSchema = z.record(z.string().min(1), ProviderConfigSchema); +export type ProvidersConfig = z.infer; +``` + +Then add to `DefaultsSchema` (after the `scan` field, around line 257): + +```ts + /** Custom OpenAI-compatible providers (e.g. self-hosted LiteLLM). Keyed by provider name. */ + providers: ProvidersConfigSchema.optional(), +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm --filter @sentry/warden test -- config/loader.test.ts -t "providers config"` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/config/schema.ts packages/warden/src/config/loader.test.ts +git commit -m "feat(config): add custom provider schema for self-hosted LLMs + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 2: Provider normalization + env key resolution + +**Files:** +- Create: `packages/warden/src/sdk/runtimes/custom-provider.ts` +- Test: `packages/warden/src/sdk/runtimes/custom-provider.test.ts` + +**Interfaces:** +- Consumes: `ProvidersConfig`, `ProviderConfig`, `ProviderModelConfig` from `../../config/schema.js`. +- Produces: + - `interface PiProviderModel { id; name; api: 'openai-completions'; reasoning: boolean; input: ('text'|'image')[]; cost: { input; output; cacheRead; cacheWrite }; contextWindow: number; maxTokens: number }` + - `interface PiProvider { name: string; baseUrl: string; api: 'openai-completions'; headers?: Record; apiKey?: string; models: PiProviderModel[] }` + - `type PiProviderOptions = { providers: PiProvider[] } | undefined` + - `function resolveProviderApiKey(name: string, apiKeyEnv: string | undefined, env: NodeJS.ProcessEnv): string | undefined` + - `function buildPiProviderOptions(providers: ProvidersConfig | undefined, env: NodeJS.ProcessEnv): PiProviderOptions` + - `function isLoopbackBaseUrl(baseUrl: string): boolean` + - `function assertCustomProviderAuth(options: PiProviderOptions): void` (throws `Error` naming the provider + expected env var) + +- [ ] **Step 1: Write the failing test** + +Create `packages/warden/src/sdk/runtimes/custom-provider.test.ts`: + +```ts +import { describe, it, expect } from 'vitest'; +import { + buildPiProviderOptions, + resolveProviderApiKey, + isLoopbackBaseUrl, + assertCustomProviderAuth, +} from './custom-provider.js'; +import type { ProvidersConfig } from '../../config/schema.js'; + +const providers: ProvidersConfig = { + litellm: { baseUrl: 'https://gw.example.com/v1', api: 'openai-completions', models: [{ id: 'my-model' }] }, +}; + +describe('resolveProviderApiKey', () => { + it('prefers apiKeyEnv when set', () => { + expect(resolveProviderApiKey('litellm', 'MY_KEY', { MY_KEY: 'k1', WARDEN_LITELLM_API_KEY: 'k2' })).toBe('k1'); + }); + it('falls back to WARDEN__API_KEY then _API_KEY', () => { + expect(resolveProviderApiKey('litellm', undefined, { WARDEN_LITELLM_API_KEY: 'k2' })).toBe('k2'); + expect(resolveProviderApiKey('litellm', undefined, { LITELLM_API_KEY: 'k3' })).toBe('k3'); + }); + it('returns undefined when nothing is set', () => { + expect(resolveProviderApiKey('litellm', undefined, {})).toBeUndefined(); + }); +}); + +describe('buildPiProviderOptions', () => { + it('returns undefined for empty input', () => { + expect(buildPiProviderOptions(undefined, {})).toBeUndefined(); + expect(buildPiProviderOptions({}, {})).toBeUndefined(); + }); + it('applies model defaults and resolves the key', () => { + const built = buildPiProviderOptions(providers, { WARDEN_LITELLM_API_KEY: 'k2' }); + const p = built?.providers[0]; + expect(p?.name).toBe('litellm'); + expect(p?.apiKey).toBe('k2'); + const m = p?.models[0]; + expect(m).toMatchObject({ + id: 'my-model', + name: 'my-model', + api: 'openai-completions', + reasoning: false, + input: ['text'], + contextWindow: 128000, + maxTokens: 8192, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }); + }); +}); + +describe('isLoopbackBaseUrl', () => { + it('recognizes loopback hosts', () => { + expect(isLoopbackBaseUrl('http://localhost:4000/v1')).toBe(true); + expect(isLoopbackBaseUrl('http://127.0.0.1:4000')).toBe(true); + expect(isLoopbackBaseUrl('https://gw.example.com/v1')).toBe(false); + }); +}); + +describe('assertCustomProviderAuth', () => { + it('throws for a non-loopback provider without a key', () => { + const built = buildPiProviderOptions(providers, {}); + expect(() => assertCustomProviderAuth(built)).toThrow(/litellm/); + }); + it('passes for a loopback provider without a key', () => { + const built = buildPiProviderOptions( + { local: { baseUrl: 'http://localhost:4000/v1', api: 'openai-completions', models: [{ id: 'm' }] } }, + {}, + ); + expect(() => assertCustomProviderAuth(built)).not.toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- runtimes/custom-provider.test.ts` +Expected: FAIL with module-not-found / export errors. + +- [ ] **Step 3: Implement the module** + +Create `packages/warden/src/sdk/runtimes/custom-provider.ts`: + +```ts +/** + * Normalization and auth resolution for custom OpenAI-compatible providers + * (e.g. self-hosted LiteLLM). Pure functions over a passed-in env object so + * they are trivially testable and never read process.env implicitly. + */ +import type { ProvidersConfig } from '../../config/schema.js'; + +const DEFAULT_CONTEXT_WINDOW = 128_000; +const DEFAULT_MAX_TOKENS = 8_192; +const DEFAULT_COST = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } as const; + +export interface PiProviderModel { + id: string; + name: string; + api: 'openai-completions'; + reasoning: boolean; + input: ('text' | 'image')[]; + cost: { input: number; output: number; cacheRead: number; cacheWrite: number }; + contextWindow: number; + maxTokens: number; +} + +export interface PiProvider { + name: string; + baseUrl: string; + api: 'openai-completions'; + headers?: Record; + apiKey?: string; + models: PiProviderModel[]; +} + +export type PiProviderOptions = { providers: PiProvider[] } | undefined; + +/** Resolve a provider API key from the environment. apiKeyEnv wins, then conventional names. */ +export function resolveProviderApiKey( + name: string, + apiKeyEnv: string | undefined, + env: NodeJS.ProcessEnv, +): string | undefined { + const upper = name.toUpperCase().replace(/[^A-Z0-9]/g, '_'); + const candidates = apiKeyEnv ? [apiKeyEnv] : [`WARDEN_${upper}_API_KEY`, `${upper}_API_KEY`]; + for (const candidate of candidates) { + const value = env[candidate]; + if (value) return value; + } + return undefined; +} + +/** Normalize warden.toml providers into Pi registerProvider input with defaults + resolved keys. */ +export function buildPiProviderOptions( + providers: ProvidersConfig | undefined, + env: NodeJS.ProcessEnv, +): PiProviderOptions { + if (!providers) return undefined; + const entries = Object.entries(providers); + if (entries.length === 0) return undefined; + + const built: PiProvider[] = entries.map(([name, config]) => ({ + name, + baseUrl: config.baseUrl, + api: config.api, + ...(config.headers ? { headers: config.headers } : {}), + apiKey: resolveProviderApiKey(name, config.apiKeyEnv, env), + models: config.models.map((model) => ({ + id: model.id, + name: model.name ?? model.id, + api: config.api, + reasoning: model.reasoning ?? false, + input: model.input ?? ['text'], + cost: model.cost ?? { ...DEFAULT_COST }, + contextWindow: model.contextWindow ?? DEFAULT_CONTEXT_WINDOW, + maxTokens: model.maxTokens ?? DEFAULT_MAX_TOKENS, + })), + })); + + return { providers: built }; +} + +/** True when the base URL points at a loopback host (unauthenticated runs allowed). */ +export function isLoopbackBaseUrl(baseUrl: string): boolean { + try { + const host = new URL(baseUrl).hostname; + return host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '[::1]'; + } catch { + return false; + } +} + +/** Fail fast when a non-loopback provider has no resolvable API key. */ +export function assertCustomProviderAuth(options: PiProviderOptions): void { + if (!options) return; + for (const provider of options.providers) { + if (!provider.apiKey && !isLoopbackBaseUrl(provider.baseUrl)) { + throw new Error( + `Custom provider "${provider.name}" has no API key. ` + + `Set WARDEN_${provider.name.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_API_KEY ` + + `(or the configured apiKeyEnv), or use a localhost baseUrl for an unauthenticated endpoint.`, + ); + } + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm --filter @sentry/warden test -- runtimes/custom-provider.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/sdk/runtimes/custom-provider.ts packages/warden/src/sdk/runtimes/custom-provider.test.ts +git commit -m "feat(runtime): normalize custom providers and resolve keys from env + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 3: Wire provider options into `getRuntimeProviderOptions` + +**Files:** +- Modify: `packages/warden/src/sdk/runtimes/index.ts:37-53` +- Modify: `packages/warden/src/sdk/runtimes/types.ts` (add `providerOptions` to auxiliary/synthesis requests) +- Test: `packages/warden/src/sdk/runtimes/index.test.ts` (exists) + +**Interfaces:** +- Consumes: `buildPiProviderOptions`, `PiProviderOptions` from `./custom-provider.js`; `ProvidersConfig` from `../../config/schema.js`. +- Produces: `RuntimeProviderOptionsInput` gains `providers?: ProvidersConfig`; `getRuntimeProviderOptions('pi', { providers })` returns `PiProviderOptions`. `AuxiliaryRunRequestBase`/`SynthesisRunRequest` gain `providerOptions?: unknown`. + +- [ ] **Step 1: Write the failing test** + +Add to `packages/warden/src/sdk/runtimes/index.test.ts`: + +```ts +import { getRuntimeProviderOptions } from './index.js'; + +describe('getRuntimeProviderOptions pi providers', () => { + it('builds pi provider options from config', () => { + const result = getRuntimeProviderOptions('pi', { + providers: { litellm: { baseUrl: 'http://localhost:4000/v1', api: 'openai-completions', models: [{ id: 'm' }] } }, + }) as { providers: { name: string; models: { id: string }[] }[] } | undefined; + expect(result?.providers[0]?.name).toBe('litellm'); + expect(result?.providers[0]?.models[0]?.id).toBe('m'); + }); + + it('returns undefined for pi without providers', () => { + expect(getRuntimeProviderOptions('pi', {})).toBeUndefined(); + }); + + it('still returns the claude executable path', () => { + expect(getRuntimeProviderOptions('claude', { pathToClaudeCodeExecutable: '/bin/claude' })) + .toEqual({ pathToClaudeCodeExecutable: '/bin/claude' }); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- runtimes/index.test.ts -t "pi providers"` +Expected: FAIL (pi branch currently returns undefined even with providers). + +- [ ] **Step 3: Implement** + +In `packages/warden/src/sdk/runtimes/index.ts`, replace the `RuntimeProviderOptionsInput` interface and `getRuntimeProviderOptions` body (lines 37-53): + +```ts +import { buildPiProviderOptions } from './custom-provider.js'; +import type { ProvidersConfig } from '../../config/schema.js'; + +export interface RuntimeProviderOptionsInput { + pathToClaudeCodeExecutable?: string; + providers?: ProvidersConfig; +} + +/** + * Build provider-specific runtime options at the runtime boundary. + */ +export function getRuntimeProviderOptions( + name: RuntimeName, + options: RuntimeProviderOptionsInput +): unknown { + if (name === 'claude') { + return { pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable }; + } + + if (name === 'pi') { + return buildPiProviderOptions(options.providers, process.env); + } + + return undefined; +} +``` + +Add the `import` lines to the top of the file with the other imports (do not duplicate the existing `claudeRuntime`/`piRuntime` imports). + +In `packages/warden/src/sdk/runtimes/types.ts`, add `providerOptions?: unknown;` to `AuxiliaryRunRequestBase` (after `maxRetries?` at line 112) and to `SynthesisRunRequest` (after `maxRetries?` at line 139), each with the doc comment: + +```ts + /** Provider-specific settings consumed only by the selected runtime adapter. */ + providerOptions?: unknown; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm --filter @sentry/warden test -- runtimes/index.test.ts -t "pi providers"` +Expected: PASS (3 tests). + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/sdk/runtimes/index.ts packages/warden/src/sdk/runtimes/types.ts packages/warden/src/sdk/runtimes/index.test.ts +git commit -m "feat(runtime): expose custom providers via getRuntimeProviderOptions + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 4: Register custom providers in the Pi adapter + +**Files:** +- Modify: `packages/warden/src/sdk/runtimes/pi.ts` (PiPromptOptions ~98-117; `runPiPrompt` registry creation ~346-348; `runSkill` request destructure ~751-796; `runStructured` call ~683-697) +- Test: `packages/warden/src/sdk/runtimes/pi.test.ts` + +**Interfaces:** +- Consumes: `PiProviderOptions` from `./custom-provider.js`; `ModelRegistry.registerProvider` and `AuthStorage.setRuntimeApiKey` from the Pi SDK. +- Produces: `runSkill`/`runStructured` honor `request.providerOptions` of shape `PiProviderOptions`; registered providers are resolvable by `resolvePiModel`. + +- [ ] **Step 1: Write the failing test** + +`pi.test.ts` mocks the Pi SDK. Inspect the existing mock setup at the top of `packages/warden/src/sdk/runtimes/pi.test.ts` and reuse it. The new test asserts that when `providerOptions` carries a provider, `registerProvider` is called with the normalized config before the model is resolved. Add: + +```ts +it('registers custom providers from providerOptions before resolving the model', async () => { + // registerProvider is a vi.fn() on the mocked ModelRegistry instance — see the + // existing mock factory in this file and extend it to capture registerProvider calls. + const calls = capturedRegisterProviderCalls; // provided by the mock factory (see Step 3) + calls.length = 0; + + await piRuntime.runAuxiliary({ + task: 'extraction', + apiKey: undefined, + prompt: 'hi', + schema: z.object({ ok: z.boolean() }), + model: 'litellm/my-model', + providerOptions: { + providers: [{ + name: 'litellm', + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions', + apiKey: 'k', + models: [{ + id: 'my-model', name: 'my-model', api: 'openai-completions', reasoning: false, + input: ['text'], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, maxTokens: 8192, + }], + }], + }, + }); + + expect(calls[0]?.[0]).toBe('litellm'); + expect(calls[0]?.[1]).toMatchObject({ baseUrl: 'http://localhost:4000/v1', apiKey: 'k' }); +}); +``` + +If the existing mock does not yet expose `registerProvider`/a capture array, that is part of Step 3. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- runtimes/pi.test.ts -t "registers custom providers"` +Expected: FAIL (registerProvider never called / mock missing). + +- [ ] **Step 3: Implement** + +In `packages/warden/src/sdk/runtimes/pi.ts`: + +(a) Add the import near the other local imports: + +```ts +import type { PiProviderOptions } from './custom-provider.js'; +``` + +(b) Add to `PiPromptOptions` (after `legacyAnthropicApiKey?` ~line 104): + +```ts + /** Custom providers to register on the model registry before resolving the model. */ + customProviders?: PiProviderOptions; +``` + +(c) Add a helper near `createAuthStorage` (after line 151): + +```ts +function registerCustomProviders( + modelRegistry: ModelRegistry, + authStorage: AuthStorage, + customProviders: PiProviderOptions, +): void { + if (!customProviders) return; + for (const provider of customProviders.providers) { + modelRegistry.registerProvider(provider.name, { + baseUrl: provider.baseUrl, + api: provider.api, + ...(provider.headers ? { headers: provider.headers } : {}), + ...(provider.apiKey ? { apiKey: provider.apiKey } : {}), + models: provider.models, + }); + if (provider.apiKey) { + authStorage.setRuntimeApiKey(provider.name, provider.apiKey); + } + } +} +``` + +(d) In `runPiPrompt`, right after `const modelRegistry = ModelRegistry.create(authStorage);` (line 347) and before `const model = resolvePiModel(...)` (line 348): + +```ts + registerCustomProviders(modelRegistry, authStorage, options.customProviders); +``` + +(e) In `runSkill` destructure `providerOptions` from `request` (add to the destructure block ~751-761), then pass it through to `runPiPrompt` (in the `runPiPrompt({ ... })` call ~783-796): + +```ts + customProviders: request.providerOptions as PiProviderOptions, +``` + +(f) In `runStructured`, the request object already spreads `...request`. Pass it into the `runPiPrompt` call (~683-697): + +```ts + customProviders: request.providerOptions as PiProviderOptions, +``` + +(`runStructured`'s `request` param type is the inline object; add `providerOptions?: unknown;` to it alongside the other fields ~643-658.) + +(g) Update the mock factory in `pi.test.ts` so the mocked `ModelRegistry` instance has `registerProvider: vi.fn((...args) => capturedRegisterProviderCalls.push(args))` and export/declare `capturedRegisterProviderCalls` at module scope, and the mocked `AuthStorage` has `setRuntimeApiKey: vi.fn()`. Mirror how the existing mock exposes `find`/`create`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm --filter @sentry/warden test -- runtimes/pi.test.ts` +Expected: PASS (new test + existing tests still green). + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/sdk/runtimes/pi.ts packages/warden/src/sdk/runtimes/pi.test.ts +git commit -m "feat(runtime): register custom providers in the pi adapter + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 5: Thread `providers` through the SDK runner + all runtime call sites + +**Files:** +- Modify: `packages/warden/src/sdk/types.ts` (`SkillRunnerOptions`, after line 116 `runtime?`) +- Modify: `packages/warden/src/sdk/analyze.ts:397-399` (agent call) and the extraction call site +- Modify: `packages/warden/src/sdk/verify.ts:268-270` (and `VerifyFindingsOptions`) +- Modify: `packages/warden/src/sdk/extract.ts:29-35` (`AuxiliaryCallOptions`) and `:230-238` +- Modify: `packages/warden/src/output/dedup.ts:616` and `:919` +- Modify: `packages/warden/src/action/fix-evaluation/judge.ts:208-229` +- Modify: `packages/warden/src/sdk/json-output.ts:72` +- Test: `packages/warden/src/sdk/extract.test.ts` (assert providerOptions is forwarded) + +**Interfaces:** +- Consumes: `getRuntimeProviderOptions` (Task 3); `ProvidersConfig` from `config/schema.js`. +- Produces: `SkillRunnerOptions.providers?: ProvidersConfig`; `AuxiliaryCallOptions.providers?: ProvidersConfig`; `VerifyFindingsOptions.providers?: ProvidersConfig`; every Pi-capable runtime request carries `providerOptions` derived from `providers`. + +- [ ] **Step 1: Write the failing test** + +In `packages/warden/src/sdk/extract.test.ts`, add a test that stubs the runtime and asserts `providerOptions` is forwarded. Reuse the file's existing runtime mock (look for where `getRuntime`/`runAuxiliary` is mocked). The assertion: + +```ts +it('forwards providerOptions built from providers to the auxiliary runtime', async () => { + // Arrange: runAuxiliary mock captures its request; runtime = 'pi'. + await extractFindingsWithLLM(rawTextWithFindings, { + runtime: 'pi', + providers: { litellm: { baseUrl: 'http://localhost:4000/v1', api: 'openai-completions', models: [{ id: 'm' }] } }, + }); + const req = runAuxiliaryMock.mock.calls[0][0]; + expect(req.providerOptions).toBeDefined(); + expect(req.providerOptions.providers[0].name).toBe('litellm'); +}); +``` + +(Adapt `rawTextWithFindings`/option shape to the existing test helpers in the file.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- extract.test.ts -t "forwards providerOptions"` +Expected: FAIL (`providers` not a known option; `providerOptions` undefined on the request). + +- [ ] **Step 3: Implement the threading** + +Make these edits (each mirrors the existing `runtime`/`auxiliaryModel` handling): + +1. `sdk/types.ts` — in `SkillRunnerOptions`, after `runtime?` (line 116): +```ts + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; +``` +Add the import: `import type { ProvidersConfig } from '../config/schema.js';` (extend the existing `config/schema.js` import on line 3). + +2. `sdk/extract.ts` — in `AuxiliaryCallOptions` (line 29-35) add: +```ts + providers?: ProvidersConfig; +``` +Import `ProvidersConfig` and `getRuntimeProviderOptions` (the latter is already re-exported via runner; import from `./runtimes/index.js`). In the `runAuxiliary` call (line 230), add: +```ts + providerOptions: getRuntimeProviderOptions(runtimeName, { providers: options.providers }), +``` + +3. `sdk/analyze.ts` — the extraction call delegates to `extractFindingsWithLLM`; ensure the options passed there include `providers: options.providers`. Find the `extractFindingsWithLLM(` / extraction-options construction in this file and add `providers: options.providers`. Also the agent `getRuntimeProviderOptions` call (line 397) becomes: +```ts + providerOptions: getRuntimeProviderOptions(runtimeName, { + pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable, + providers: options.providers, + }), +``` + +4. `sdk/verify.ts` — add `providers?: ProvidersConfig` to `VerifyFindingsOptions`; line 268 call becomes: +```ts + providerOptions: getRuntimeProviderOptions(runtimeName, { + pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable, + providers: options.providers, + }), +``` + +5. `output/dedup.ts` — add `providers?: ProvidersConfig` to the dedup options type; both `runAuxiliary` calls (lines 616, 919) add: +```ts + providerOptions: getRuntimeProviderOptions(options.runtime ?? 'claude', { providers: options.providers }), +``` + +6. `action/fix-evaluation/judge.ts` — add `providers?: ProvidersConfig` to `runtimeOptions`; the `runAuxiliary` call (line 226) adds: +```ts + providerOptions: getRuntimeProviderOptions(runtimeOptions.runtime, { providers: runtimeOptions.providers }), +``` + +7. `sdk/json-output.ts` — the repair path (line 72) builds a runtime request; add `providerOptions: getRuntimeProviderOptions(repair.runtimeName ?? 'claude', { providers: repair.providers })` to that request and `providers?: ProvidersConfig` to the repair options type. + +For each, import `getRuntimeProviderOptions` from `./runtimes/index.js` (or `../sdk/runtimes/index.js` for non-sdk files) and `ProvidersConfig` from the appropriate `config/schema.js` relative path. + +- [ ] **Step 4: Run tests** + +Run: `pnpm --filter @sentry/warden test -- extract.test.ts -t "forwards providerOptions"` → PASS. +Then `pnpm --filter @sentry/warden typecheck` → no errors (catches any missed `providers` field or import). + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/sdk packages/warden/src/output/dedup.ts packages/warden/src/action/fix-evaluation/judge.ts +git commit -m "feat(runtime): thread custom providers through all model lanes + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 6: Resolve `providers` from config and pass it to every runner entry point + +**Files:** +- Modify: `packages/warden/src/config/loader.ts` (`ResolvedTrigger` ~400, `resolveSkillConfigs` ~495 + both push branches ~546-556 and ~581-592) +- Modify: `packages/warden/src/action/triggers/executor.ts:189-192` +- Modify: `packages/warden/src/action/workflow/schedule.ts:216-219` +- Modify: `packages/warden/src/action/workflow/pr-workflow.ts:184` + auxiliary/fix-eval option construction (~313, ~566, ~635, ~685) +- Modify: `packages/warden/src/action/review/poster.ts:255-315` (consolidate + dedup option construction) +- Modify: `packages/warden/src/cli/main.ts:1100-1130, 1153-1171` (runner options + per-trigger options) +- Test: `packages/warden/src/config/loader.test.ts` + +**Interfaces:** +- Consumes: `WardenConfig.defaults.providers` (Task 1). +- Produces: `ResolvedTrigger.providers?: ProvidersConfig`, populated from `defaults.providers`; carried into `SkillRunnerOptions.providers` and every auxiliary/synthesis options object. + +- [ ] **Step 1: Write the failing test** + +In `packages/warden/src/config/loader.test.ts`, add: + +```ts +it('propagates defaults.providers onto every resolved trigger', () => { + const config = WardenConfigSchema.parse({ + version: 1, + defaults: { providers: { litellm: { baseUrl: 'http://localhost:4000/v1', models: [{ id: 'm' }] } } }, + skills: [{ name: 'a' }, { name: 'b', triggers: [{ type: 'local' }] }], + }); + const resolved = resolveSkillConfigs(config); + expect(resolved).toHaveLength(2); + for (const trigger of resolved) { + expect(trigger.providers?.['litellm']?.baseUrl).toBe('http://localhost:4000/v1'); + } +}); +``` + +(Reuse the existing `resolveSkillConfigs`/`WardenConfigSchema` imports already present in the test file.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- config/loader.test.ts -t "propagates defaults.providers"` +Expected: FAIL (`trigger.providers` is undefined). + +- [ ] **Step 3: Implement** + +1. `config/loader.ts`: + - Add to `ResolvedTrigger` (after `runtime?` ~line 401): `providers?: ProvidersConfig;` + - In `resolveSkillConfigs`, after `const runtime = defaults?.runtime ?? 'pi';` (line 495): `const providers = defaults?.providers;` + - Add `providers,` to both pushed objects (next to `runtime,` at lines ~546 and ~581). + - Import `ProvidersConfig` from `./schema.js` (extend the existing schema import). + +2. Each runner entry point copies `providers` next to where it already copies `auxiliaryModel`/`runtime` into the options object it builds: + - `action/triggers/executor.ts:189-192` → add `providers: trigger.providers,` + - `action/workflow/schedule.ts:216-219` → add `providers: resolved.providers,` + - `action/workflow/pr-workflow.ts` → set `providers` on the runner options (near line 184) from `baseDefaults?.providers ?? repoDefaults?.providers`, and on `auxiliaryOptions` (line 313) + fix-eval options (lines 635/685) carry the same `providers`. + - `action/review/poster.ts` → the consolidate options (line 282) and dedup options (line 315) add `providers: ctx.providers` (add `providers?: ProvidersConfig` to the poster `ctx` type, populated upstream from the resolved trigger). + - `cli/main.ts` → add `providers: match?.providers ?? config?.defaults?.providers` to the runner options near line 1100, and `providers: t.providers` to the per-trigger options near line 1127; add `providers` to the local options interface (~615) and the `mergeOptions` allow-list (~646) if model fields are gated there. + +- [ ] **Step 4: Run tests + typecheck** + +Run: `pnpm --filter @sentry/warden test -- config/loader.test.ts -t "propagates defaults.providers"` → PASS. +Run: `pnpm --filter @sentry/warden typecheck` → no errors (catches any entry point still missing `providers` where the options type now requires it, or unused-field issues). + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/config/loader.ts packages/warden/src/action packages/warden/src/cli/main.ts packages/warden/src/config/loader.test.ts +git commit -m "feat(config): propagate custom providers to all runner entry points + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 7: Preflight auth validation for custom providers + +**Files:** +- Modify: `packages/warden/src/cli/main.ts:778-790` (the preflight auth path that already calls `verifyAuth`) +- Modify: `packages/warden/src/action/workflow/base.ts` (or the action preflight near the existing auth guard ~line 71) +- Test: `packages/warden/src/cli/main.test.ts` if present, else assert via a focused unit on `assertCustomProviderAuth` already covered in Task 2 — add one integration assertion in `cli/main.test.ts` only if the harness exists. + +**Interfaces:** +- Consumes: `assertCustomProviderAuth`, `buildPiProviderOptions` from `../sdk/runtimes/custom-provider.js`. +- Produces: a clear thrown error before analysis starts when a non-loopback custom provider has no key. + +- [ ] **Step 1: Write the failing test** + +If `cli/main.test.ts` exists with a preflight harness, add: + +```ts +it('fails preflight when a non-loopback custom provider has no key', async () => { + // Arrange runner options with runtime 'pi' and providers pointing at a remote URL, + // with the relevant env var unset. Expect the run to reject with /API key/. + await expect(runWithOptions({ + runtime: 'pi', + providers: { litellm: { baseUrl: 'https://gw.example.com/v1', models: [{ id: 'm' }] } }, + })).rejects.toThrow(/litellm/); +}); +``` + +If no such harness exists, skip the integration test (Task 2 already unit-tests `assertCustomProviderAuth`) and proceed to Step 3; record this in the commit body. + +- [ ] **Step 2: Run test to verify it fails (if written)** + +Run: `pnpm --filter @sentry/warden test -- cli/main.test.ts -t "fails preflight"` +Expected: FAIL (no preflight check yet). + +- [ ] **Step 3: Implement** + +In the CLI preflight (`cli/main.ts`, the function around line 778-790 that calls `verifyAuth`), after the existing auth handling, add (only when `runtime === 'pi'`): + +```ts +import { buildPiProviderOptions, assertCustomProviderAuth } from '../sdk/runtimes/custom-provider.js'; + +// ...inside the preflight, when options.runtime is 'pi': +assertCustomProviderAuth(buildPiProviderOptions(options.providers, process.env)); +``` + +Add the equivalent guard in the action preflight next to the existing API-key error in `action/workflow/base.ts` (~line 71), guarded by `runtime === 'pi'`. + +- [ ] **Step 4: Run tests** + +Run: `pnpm --filter @sentry/warden test -- cli` and `pnpm --filter @sentry/warden typecheck` → PASS / no errors. + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/cli/main.ts packages/warden/src/action/workflow/base.ts +git commit -m "feat(runtime): fail fast when a remote custom provider has no key + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 8: Documentation + +**Files:** +- Modify: `packages/docs/src/content/docs/config/models.mdx` (after the "Pi Model Selectors" section, before "Claude Runtime Models" ~line 58) + +**Interfaces:** none (docs only). + +- [ ] **Step 1: Add the docs section** + +Insert into `packages/docs/src/content/docs/config/models.mdx`: + +````mdx +## Self-hosted / OpenAI-compatible providers (LiteLLM) + +When `runtime = "pi"`, you can register a self-hosted, OpenAI-compatible endpoint +(such as a [LiteLLM](https://docs.litellm.ai/) proxy) as a named provider and +target its models with the usual `provider/model` selector. The custom provider +covers every model lane (agent, auxiliary, synthesis). + +```toml title="warden.toml" +[defaults] +runtime = "pi" + +[defaults.providers.litellm] +baseUrl = "http://localhost:4000/v1" # required; OpenAI-compatible base URL +api = "openai-completions" # optional; default +# headers = { "X-Tenant" = "team-a" } # optional custom headers +# apiKeyEnv = "WARDEN_LITELLM_API_KEY" # optional; overrides the default lookup + +[[defaults.providers.litellm.models]] +id = "my-model" # required; the model name your endpoint exposes +# contextWindow = 128000 # optional; default 128000 +# maxTokens = 8192 # optional; default 8192 +# reasoning = false # optional; default false +# cost = { input = 0, output = 0, cacheRead = 0, cacheWrite = 0 } + +[defaults.agent] +model = "litellm/my-model" +``` + +**Model defaults:** only `id` is required. Warden fills `name` (= `id`), +`reasoning` (`false`), `input` (`["text"]`), `contextWindow` (`128000`), +`maxTokens` (`8192`), and `cost` (all zeros). Self-hosted runs therefore report +`$0` cost unless you set explicit costs. + +**Authentication:** the API key is read from the environment, never from +`warden.toml`. Warden looks up `apiKeyEnv` if set, otherwise +`WARDEN__API_KEY` then `_API_KEY` (e.g. `WARDEN_LITELLM_API_KEY`). +A `localhost` / `127.0.0.1` base URL may run without a key; any other host +requires one or Warden fails before analysis starts. + +### Alternative: Pi `models.json` + +Advanced users can instead define a custom provider in Pi's own `models.json` +(the format Pi loads on startup). Warden's Pi runtime picks up any providers and +models defined there. Prefer the `warden.toml` block above unless you already +maintain a shared Pi configuration. +```` + +- [ ] **Step 2: Build the docs** + +Run: `pnpm --filter warden-docs build` (or the repo's docs build script) +Expected: build succeeds. + +- [ ] **Step 3: Commit** + +```bash +git add packages/docs/src/content/docs/config/models.mdx +git commit -m "docs: document self-hosted OpenAI-compatible providers (LiteLLM) + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Final verification + +- [ ] Run `pnpm lint && pnpm build && pnpm test` from the repo root. All green. +- [ ] Manual smoke (optional, requires a reachable LiteLLM): set `WARDEN_LITELLM_API_KEY`, add the `[defaults.providers.litellm]` block with `model = "litellm/"`, run `warden run` on a small diff, confirm analysis completes against the custom endpoint. + +--- + +## Self-Review + +**Spec coverage:** +- Config surface (`[defaults.providers.*]`) → Task 1. ✓ +- Model defaults (only `id` required) → Task 2 (`buildPiProviderOptions`). ✓ +- Auth from env, no secrets in config, loopback exception, fail-fast preflight → Task 2 (`resolveProviderApiKey`/`assertCustomProviderAuth`) + Task 7. ✓ +- `getRuntimeProviderOptions('pi', …)` → Task 3. ✓ +- Pi adapter `registerProvider` at the `runPiPrompt` choke point → Task 4. ✓ +- All lanes (agent + auxiliary + synthesis) → Tasks 3 (request types) + 5 (call sites) + 6 (entry points). ✓ +- `models.json` passthrough documented → Task 8. ✓ +- Tests (schema, builder, provider options, registration, threading, propagation) → Tasks 1-6. ✓ +- Docs → Task 8. ✓ +- Out of scope (per-skill providers, ANTHROPIC_BASE_URL formalization, non-OpenAI compat) → not implemented, consistent with spec. ✓ + +**Placeholder scan:** No "TBD"/"implement later". The two places that say "if the harness exists" (Task 5 mock extension, Task 7 integration test) are conditional on existing test infrastructure and give a concrete fallback; not placeholders for missing plan content. + +**Type consistency:** `ProvidersConfig`/`ProviderConfig`/`ProviderModelConfig` (Task 1) are used verbatim in Tasks 2/3/5/6. `PiProviderOptions`/`PiProvider`/`PiProviderModel` (Task 2) are used verbatim in Tasks 3/4. `getRuntimeProviderOptions(name, { providers })` signature (Task 3) matches every call site in Tasks 5/7. `request.providerOptions` (Task 3 types) is read in Task 4 and written in Task 5. `registerProvider(name, { baseUrl, api, headers, apiKey, models })` matches the verified `ProviderConfigInput` shape. From ff7edb0475fbb63db9787d41c2cdf2f346532dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Fri, 26 Jun 2026 23:52:07 +0200 Subject: [PATCH 03/22] feat(config): add custom provider schema for self-hosted LLMs Co-Authored-By: Claude Opus 4.8 --- packages/warden/src/config/loader.test.ts | 48 +++++++++++++++++++++++ packages/warden/src/config/schema.ts | 46 ++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/packages/warden/src/config/loader.test.ts b/packages/warden/src/config/loader.test.ts index b6a443f6..dd8fe3c5 100644 --- a/packages/warden/src/config/loader.test.ts +++ b/packages/warden/src/config/loader.test.ts @@ -1644,3 +1644,51 @@ describe('logs config', () => { expect(result.data?.logs).toBeUndefined(); }); }); + +describe('providers config', () => { + const base = { version: 1 as const, skills: [] }; + + it('accepts a valid custom provider', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { + providers: { + litellm: { + baseUrl: 'http://localhost:4000/v1', + models: [{ id: 'my-model' }], + }, + }, + }, + }); + expect(result.success).toBe(true); + if (result.success) { + const p = result.data.defaults?.providers?.['litellm']; + expect(p?.api).toBe('openai-completions'); // default applied + expect(p?.models[0]?.id).toBe('my-model'); + } + }); + + it('rejects a non-URL baseUrl', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { litellm: { baseUrl: 'not a url', models: [{ id: 'm' }] } } }, + }); + expect(result.success).toBe(false); + }); + + it('rejects an unsupported api value', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { x: { baseUrl: 'http://h/v1', api: 'cohere', models: [{ id: 'm' }] } } }, + }); + expect(result.success).toBe(false); + }); + + it('rejects an empty models array', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { x: { baseUrl: 'http://h/v1', models: [] } } }, + }); + expect(result.success).toBe(false); + }); +}); diff --git a/packages/warden/src/config/schema.ts b/packages/warden/src/config/schema.ts index 245c30df..84563b4e 100644 --- a/packages/warden/src/config/schema.ts +++ b/packages/warden/src/config/schema.ts @@ -216,6 +216,50 @@ export const DEFAULT_SCAN_LIMITS: Required = { maxFileLines: 3_000, }; +// Per-model definition for a custom provider. Only `id` is required; Warden +// fills the remaining fields Pi requires with sensible defaults. +export const ProviderModelConfigSchema = z.object({ + /** Model id as exposed by the endpoint (e.g. the LiteLLM model name). */ + id: z.string().min(1), + /** Display name. Defaults to `id`. */ + name: z.string().min(1).optional(), + /** Whether the model supports reasoning/thinking. Default: false. */ + reasoning: z.boolean().optional(), + /** Accepted input modalities. Default: ["text"]. */ + input: z.array(z.enum(['text', 'image'])).min(1).optional(), + /** Context window in tokens. Default: 128000. */ + contextWindow: z.number().int().positive().optional(), + /** Max output tokens. Default: 8192. */ + maxTokens: z.number().int().positive().optional(), + /** Per-token cost (USD per token). Default: all zeros. */ + cost: z.object({ + input: z.number().nonnegative(), + output: z.number().nonnegative(), + cacheRead: z.number().nonnegative(), + cacheWrite: z.number().nonnegative(), + }).optional(), +}).strict(); +export type ProviderModelConfig = z.infer; + +// A custom, self-hosted or gateway provider (e.g. LiteLLM). OpenAI-compatible only in v1. +export const ProviderConfigSchema = z.object({ + /** OpenAI-compatible base URL, typically ending in /v1. */ + baseUrl: z.string().url(), + /** Wire protocol. Only "openai-completions" is supported today. */ + api: z.enum(['openai-completions']).default('openai-completions'), + /** Extra HTTP headers sent on every request. */ + headers: z.record(z.string(), z.string()).optional(), + /** Env var holding the API key. Defaults to WARDEN__API_KEY then _API_KEY. */ + apiKeyEnv: z.string().min(1).optional(), + /** Models this provider exposes. At least one is required. */ + models: z.array(ProviderModelConfigSchema).min(1), +}).strict(); +export type ProviderConfig = z.infer; + +// Map of provider name -> provider config. The name becomes the model-selector prefix. +export const ProvidersConfigSchema = z.record(z.string().min(1), ProviderConfigSchema); +export type ProvidersConfig = z.infer; + // Default configuration that skills inherit from export const DefaultsSchema = z.object({ /** Fail the build when findings meet this severity */ @@ -255,6 +299,8 @@ export const DefaultsSchema = z.object({ ignore: IgnoreConfigSchema.optional(), /** Global scan limits applied after ignore filtering */ scan: ScanConfigSchema.optional(), + /** Custom OpenAI-compatible providers (e.g. self-hosted LiteLLM). Keyed by provider name. */ + providers: ProvidersConfigSchema.optional(), /** Delay in milliseconds between batch starts when processing files in parallel. Default: 0 */ batchDelayMs: z.number().int().nonnegative().optional(), /** Max retries for auxiliary structured model calls (extraction repair, merging, dedup, fix evaluation). Default: 5 */ From 4e24bbc5cf8c521103b0d9fbba14b1e90c501c1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Fri, 26 Jun 2026 23:56:09 +0200 Subject: [PATCH 04/22] feat(runtime): normalize custom providers and resolve keys from env Co-Authored-By: Claude Opus 4.8 --- .../src/sdk/runtimes/custom-provider.test.ts | 71 ++++++++++++ .../src/sdk/runtimes/custom-provider.ts | 101 ++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 packages/warden/src/sdk/runtimes/custom-provider.test.ts create mode 100644 packages/warden/src/sdk/runtimes/custom-provider.ts diff --git a/packages/warden/src/sdk/runtimes/custom-provider.test.ts b/packages/warden/src/sdk/runtimes/custom-provider.test.ts new file mode 100644 index 00000000..0fa82f8a --- /dev/null +++ b/packages/warden/src/sdk/runtimes/custom-provider.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect } from 'vitest'; +import { + buildPiProviderOptions, + resolveProviderApiKey, + isLoopbackBaseUrl, + assertCustomProviderAuth, +} from './custom-provider.js'; +import type { ProvidersConfig } from '../../config/schema.js'; + +const providers: ProvidersConfig = { + litellm: { baseUrl: 'https://gw.example.com/v1', api: 'openai-completions', models: [{ id: 'my-model' }] }, +}; + +describe('resolveProviderApiKey', () => { + it('prefers apiKeyEnv when set', () => { + expect(resolveProviderApiKey('litellm', 'MY_KEY', { MY_KEY: 'k1', WARDEN_LITELLM_API_KEY: 'k2' })).toBe('k1'); + }); + it('falls back to WARDEN__API_KEY then _API_KEY', () => { + expect(resolveProviderApiKey('litellm', undefined, { WARDEN_LITELLM_API_KEY: 'k2' })).toBe('k2'); + expect(resolveProviderApiKey('litellm', undefined, { LITELLM_API_KEY: 'k3' })).toBe('k3'); + }); + it('returns undefined when nothing is set', () => { + expect(resolveProviderApiKey('litellm', undefined, {})).toBeUndefined(); + }); +}); + +describe('buildPiProviderOptions', () => { + it('returns undefined for empty input', () => { + expect(buildPiProviderOptions(undefined, {})).toBeUndefined(); + expect(buildPiProviderOptions({}, {})).toBeUndefined(); + }); + it('applies model defaults and resolves the key', () => { + const built = buildPiProviderOptions(providers, { WARDEN_LITELLM_API_KEY: 'k2' }); + const p = built?.providers[0]; + expect(p?.name).toBe('litellm'); + expect(p?.apiKey).toBe('k2'); + const m = p?.models[0]; + expect(m).toMatchObject({ + id: 'my-model', + name: 'my-model', + api: 'openai-completions', + reasoning: false, + input: ['text'], + contextWindow: 128000, + maxTokens: 8192, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }); + }); +}); + +describe('isLoopbackBaseUrl', () => { + it('recognizes loopback hosts', () => { + expect(isLoopbackBaseUrl('http://localhost:4000/v1')).toBe(true); + expect(isLoopbackBaseUrl('http://127.0.0.1:4000')).toBe(true); + expect(isLoopbackBaseUrl('https://gw.example.com/v1')).toBe(false); + }); +}); + +describe('assertCustomProviderAuth', () => { + it('throws for a non-loopback provider without a key', () => { + const built = buildPiProviderOptions(providers, {}); + expect(() => assertCustomProviderAuth(built)).toThrow(/litellm/); + }); + it('passes for a loopback provider without a key', () => { + const built = buildPiProviderOptions( + { local: { baseUrl: 'http://localhost:4000/v1', api: 'openai-completions', models: [{ id: 'm' }] } }, + {}, + ); + expect(() => assertCustomProviderAuth(built)).not.toThrow(); + }); +}); diff --git a/packages/warden/src/sdk/runtimes/custom-provider.ts b/packages/warden/src/sdk/runtimes/custom-provider.ts new file mode 100644 index 00000000..4529e6d7 --- /dev/null +++ b/packages/warden/src/sdk/runtimes/custom-provider.ts @@ -0,0 +1,101 @@ +/** + * Normalization and auth resolution for custom OpenAI-compatible providers + * (e.g. self-hosted LiteLLM). Pure functions over a passed-in env object so + * they are trivially testable and never read process.env implicitly. + */ +import type { ProvidersConfig } from '../../config/schema.js'; + +const DEFAULT_CONTEXT_WINDOW = 128_000; +const DEFAULT_MAX_TOKENS = 8_192; +const DEFAULT_COST = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } as const; + +export interface PiProviderModel { + id: string; + name: string; + api: 'openai-completions'; + reasoning: boolean; + input: ('text' | 'image')[]; + cost: { input: number; output: number; cacheRead: number; cacheWrite: number }; + contextWindow: number; + maxTokens: number; +} + +export interface PiProvider { + name: string; + baseUrl: string; + api: 'openai-completions'; + headers?: Record; + apiKey?: string; + models: PiProviderModel[]; +} + +export type PiProviderOptions = { providers: PiProvider[] } | undefined; + +/** Resolve a provider API key from the environment. apiKeyEnv wins, then conventional names. */ +export function resolveProviderApiKey( + name: string, + apiKeyEnv: string | undefined, + env: NodeJS.ProcessEnv, +): string | undefined { + const upper = name.toUpperCase().replace(/[^A-Z0-9]/g, '_'); + const candidates = apiKeyEnv ? [apiKeyEnv] : [`WARDEN_${upper}_API_KEY`, `${upper}_API_KEY`]; + for (const candidate of candidates) { + const value = env[candidate]; + if (value) return value; + } + return undefined; +} + +/** Normalize warden.toml providers into Pi registerProvider input with defaults + resolved keys. */ +export function buildPiProviderOptions( + providers: ProvidersConfig | undefined, + env: NodeJS.ProcessEnv, +): PiProviderOptions { + if (!providers) return undefined; + const entries = Object.entries(providers); + if (entries.length === 0) return undefined; + + const built: PiProvider[] = entries.map(([name, config]) => ({ + name, + baseUrl: config.baseUrl, + api: config.api, + ...(config.headers ? { headers: config.headers } : {}), + apiKey: resolveProviderApiKey(name, config.apiKeyEnv, env), + models: config.models.map((model) => ({ + id: model.id, + name: model.name ?? model.id, + api: config.api, + reasoning: model.reasoning ?? false, + input: model.input ?? ['text'], + cost: model.cost ?? { ...DEFAULT_COST }, + contextWindow: model.contextWindow ?? DEFAULT_CONTEXT_WINDOW, + maxTokens: model.maxTokens ?? DEFAULT_MAX_TOKENS, + })), + })); + + return { providers: built }; +} + +/** True when the base URL points at a loopback host (unauthenticated runs allowed). */ +export function isLoopbackBaseUrl(baseUrl: string): boolean { + try { + const host = new URL(baseUrl).hostname; + return host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '[::1]'; + } catch { + return false; + } +} + +/** Fail fast when a non-loopback provider has no resolvable API key. */ +export function assertCustomProviderAuth(options: PiProviderOptions): void { + if (!options) return; + for (const provider of options.providers) { + if (!provider.apiKey && !isLoopbackBaseUrl(provider.baseUrl)) { + throw new Error( + `Custom provider "${provider.name}" has no API key. ` + + `Set WARDEN_${provider.name.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_API_KEY ` + + `(or the configured apiKeyEnv), or use a localhost baseUrl for an unauthenticated endpoint.`, + ); + } + } +} From 4263424ed52e5d62f2cec46499f7103fe57b99cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Fri, 26 Jun 2026 23:59:48 +0200 Subject: [PATCH 05/22] fix(runtime): drop unreachable IPv6 branch, dedupe provider-name sanitize Co-Authored-By: Claude Haiku 4.5 --- packages/warden/src/sdk/runtimes/custom-provider.test.ts | 3 +++ packages/warden/src/sdk/runtimes/custom-provider.ts | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/warden/src/sdk/runtimes/custom-provider.test.ts b/packages/warden/src/sdk/runtimes/custom-provider.test.ts index 0fa82f8a..22b0b415 100644 --- a/packages/warden/src/sdk/runtimes/custom-provider.test.ts +++ b/packages/warden/src/sdk/runtimes/custom-provider.test.ts @@ -54,6 +54,9 @@ describe('isLoopbackBaseUrl', () => { expect(isLoopbackBaseUrl('http://127.0.0.1:4000')).toBe(true); expect(isLoopbackBaseUrl('https://gw.example.com/v1')).toBe(false); }); + it('recognizes bracketed IPv6 loopback', () => { + expect(isLoopbackBaseUrl('http://[::1]:4000/v1')).toBe(true); + }); }); describe('assertCustomProviderAuth', () => { diff --git a/packages/warden/src/sdk/runtimes/custom-provider.ts b/packages/warden/src/sdk/runtimes/custom-provider.ts index 4529e6d7..11f281b5 100644 --- a/packages/warden/src/sdk/runtimes/custom-provider.ts +++ b/packages/warden/src/sdk/runtimes/custom-provider.ts @@ -9,6 +9,10 @@ const DEFAULT_CONTEXT_WINDOW = 128_000; const DEFAULT_MAX_TOKENS = 8_192; const DEFAULT_COST = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } as const; +function sanitizeProviderName(name: string): string { + return name.toUpperCase().replace(/[^A-Z0-9]/g, '_'); +} + export interface PiProviderModel { id: string; name: string; @@ -37,7 +41,7 @@ export function resolveProviderApiKey( apiKeyEnv: string | undefined, env: NodeJS.ProcessEnv, ): string | undefined { - const upper = name.toUpperCase().replace(/[^A-Z0-9]/g, '_'); + const upper = sanitizeProviderName(name); const candidates = apiKeyEnv ? [apiKeyEnv] : [`WARDEN_${upper}_API_KEY`, `${upper}_API_KEY`]; for (const candidate of candidates) { const value = env[candidate]; @@ -93,7 +97,7 @@ export function assertCustomProviderAuth(options: PiProviderOptions): void { if (!provider.apiKey && !isLoopbackBaseUrl(provider.baseUrl)) { throw new Error( `Custom provider "${provider.name}" has no API key. ` + - `Set WARDEN_${provider.name.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_API_KEY ` + + `Set WARDEN_${sanitizeProviderName(provider.name)}_API_KEY ` + `(or the configured apiKeyEnv), or use a localhost baseUrl for an unauthenticated endpoint.`, ); } From 8b116e45ffe0dc28e5a01cababaa2bb63f9df45e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Sat, 27 Jun 2026 00:02:33 +0200 Subject: [PATCH 06/22] fix(runtime): keep bracketed IPv6 loopback branch (URL.hostname brackets ::1) The re-review premise was inverted: Node's WHATWG URL returns IPv6 hosts bracketed ([::1]), so the [::1] branch is the working one. Restore it and document the behavior; keep ::1 as a defensive fallback. Co-Authored-By: Claude Opus 4.8 --- packages/warden/src/sdk/runtimes/custom-provider.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/warden/src/sdk/runtimes/custom-provider.ts b/packages/warden/src/sdk/runtimes/custom-provider.ts index 11f281b5..51dd056a 100644 --- a/packages/warden/src/sdk/runtimes/custom-provider.ts +++ b/packages/warden/src/sdk/runtimes/custom-provider.ts @@ -84,7 +84,9 @@ export function buildPiProviderOptions( export function isLoopbackBaseUrl(baseUrl: string): boolean { try { const host = new URL(baseUrl).hostname; - return host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '[::1]'; + // Node's WHATWG URL returns IPv6 hosts bracketed (e.g. `[::1]`); the + // unbracketed `::1` is kept as a defensive fallback for pre-normalized input. + return host === 'localhost' || host === '127.0.0.1' || host === '[::1]' || host === '::1'; } catch { return false; } From 43c4c164ade7747d43a97d46f24ae166e0392caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Sat, 27 Jun 2026 00:05:07 +0200 Subject: [PATCH 07/22] feat(runtime): expose custom providers via getRuntimeProviderOptions Co-Authored-By: Claude Opus 4.8 --- .../warden/src/sdk/runtimes/index.test.ts | 19 +++++++++++++++++++ packages/warden/src/sdk/runtimes/index.ts | 7 +++++++ packages/warden/src/sdk/runtimes/types.ts | 4 ++++ 3 files changed, 30 insertions(+) diff --git a/packages/warden/src/sdk/runtimes/index.test.ts b/packages/warden/src/sdk/runtimes/index.test.ts index 4ed4c278..9d38cbd9 100644 --- a/packages/warden/src/sdk/runtimes/index.test.ts +++ b/packages/warden/src/sdk/runtimes/index.test.ts @@ -7,6 +7,25 @@ import { piRuntime, } from './index.js'; +describe('getRuntimeProviderOptions pi providers', () => { + it('builds pi provider options from config', () => { + const result = getRuntimeProviderOptions('pi', { + providers: { litellm: { baseUrl: 'http://localhost:4000/v1', api: 'openai-completions', models: [{ id: 'm' }] } }, + }) as { providers: { name: string; models: { id: string }[] }[] } | undefined; + expect(result?.providers[0]?.name).toBe('litellm'); + expect(result?.providers[0]?.models[0]?.id).toBe('m'); + }); + + it('returns undefined for pi without providers', () => { + expect(getRuntimeProviderOptions('pi', {})).toBeUndefined(); + }); + + it('still returns the claude executable path', () => { + expect(getRuntimeProviderOptions('claude', { pathToClaudeCodeExecutable: '/bin/claude' })) + .toEqual({ pathToClaudeCodeExecutable: '/bin/claude' }); + }); +}); + describe('runtimes', () => { it('exposes Pi as the default runtime provider', () => { const runtime = getRuntime(); diff --git a/packages/warden/src/sdk/runtimes/index.ts b/packages/warden/src/sdk/runtimes/index.ts index 93a53996..90d46cfe 100644 --- a/packages/warden/src/sdk/runtimes/index.ts +++ b/packages/warden/src/sdk/runtimes/index.ts @@ -1,6 +1,8 @@ import { claudeRuntime } from './claude.js'; import { piRuntime } from './pi.js'; import type { Runtime, RuntimeName } from './types.js'; +import { buildPiProviderOptions } from './custom-provider.js'; +import type { ProvidersConfig } from '../../config/schema.js'; const RUNTIMES: Partial> = { claude: claudeRuntime, @@ -36,6 +38,7 @@ export function getRuntime(name: RuntimeName = 'pi'): Runtime { export interface RuntimeProviderOptionsInput { pathToClaudeCodeExecutable?: string; + providers?: ProvidersConfig; } /** @@ -49,5 +52,9 @@ export function getRuntimeProviderOptions( return { pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable }; } + if (name === 'pi') { + return buildPiProviderOptions(options.providers, process.env); + } + return undefined; } diff --git a/packages/warden/src/sdk/runtimes/types.ts b/packages/warden/src/sdk/runtimes/types.ts index e1a16eb5..e2ec9721 100644 --- a/packages/warden/src/sdk/runtimes/types.ts +++ b/packages/warden/src/sdk/runtimes/types.ts @@ -110,6 +110,8 @@ interface AuxiliaryRunRequestBase { maxTokens?: number; timeout?: number; maxRetries?: number; + /** Provider-specific settings consumed only by the selected runtime adapter. */ + providerOptions?: unknown; } interface AuxiliaryRunRequestWithoutTools extends AuxiliaryRunRequestBase { @@ -137,6 +139,8 @@ export interface SynthesisRunRequest { maxTokens?: number; timeout?: number; maxRetries?: number; + /** Provider-specific settings consumed only by the selected runtime adapter. */ + providerOptions?: unknown; } export interface Runtime { From fc1c463a1c709c031c2fcca47054f07a4f68da8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Sat, 27 Jun 2026 00:10:06 +0200 Subject: [PATCH 08/22] feat(runtime): register custom providers in the pi adapter Co-Authored-By: Claude Opus 4.8 --- packages/warden/src/sdk/runtimes/pi.test.ts | 30 +++++++++++++++++++++ packages/warden/src/sdk/runtimes/pi.ts | 28 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/packages/warden/src/sdk/runtimes/pi.test.ts b/packages/warden/src/sdk/runtimes/pi.test.ts index 21585ba3..3fe29d0a 100644 --- a/packages/warden/src/sdk/runtimes/pi.test.ts +++ b/packages/warden/src/sdk/runtimes/pi.test.ts @@ -25,6 +25,7 @@ const piMocks = vi.hoisted(() => { const registry = { find: vi.fn((_provider: string, _modelId: string) => model), getAll: vi.fn(() => [model]), + registerProvider: vi.fn(), }; const session = { sessionId: 'pi-session-1', @@ -540,4 +541,33 @@ describe('piRuntime structured calls', () => { expect(result.error).toContain('Validation failed'); } }); + + it('registers custom providers from providerOptions before resolving the model', async () => { + await piRuntime.runAuxiliary({ + task: 'extraction', + apiKey: undefined, + prompt: 'hi', + schema: z.object({ ok: z.boolean() }), + model: 'litellm/my-model', + providerOptions: { + providers: [{ + name: 'litellm', + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions', + apiKey: 'k', + models: [{ + id: 'my-model', name: 'my-model', api: 'openai-completions', reasoning: false, + input: ['text'], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, maxTokens: 8192, + }], + }], + }, + }); + + expect(piMocks.registry.registerProvider).toHaveBeenCalledWith( + 'litellm', + expect.objectContaining({ baseUrl: 'http://localhost:4000/v1', apiKey: 'k' }), + ); + expect(piMocks.authStorage.setRuntimeApiKey).toHaveBeenCalledWith('litellm', 'k'); + }); }); diff --git a/packages/warden/src/sdk/runtimes/pi.ts b/packages/warden/src/sdk/runtimes/pi.ts index 540b1601..1a60128a 100644 --- a/packages/warden/src/sdk/runtimes/pi.ts +++ b/packages/warden/src/sdk/runtimes/pi.ts @@ -51,6 +51,7 @@ import { setGenAiUsageAttrs, } from '../otel.js'; import { aggregateUsage, emptyUsage } from '../usage.js'; +import type { PiProviderOptions } from './custom-provider.js'; import { InvalidPiModelSelectorError, isPiModelSelector } from './model-selectors.js'; import type { AuxiliaryRunRequest, @@ -102,6 +103,8 @@ interface PiPromptOptions { agentName?: string; model?: string; legacyAnthropicApiKey?: string; + /** Custom providers to register on the model registry before resolving the model. */ + customProviders?: PiProviderOptions; toolNames: string[]; customTools?: ToolDefinition[]; maxTurns?: number; @@ -150,6 +153,26 @@ function createAuthStorage(model: string | undefined, legacyAnthropicApiKey: str return authStorage; } +function registerCustomProviders( + modelRegistry: ModelRegistry, + authStorage: AuthStorage, + customProviders: PiProviderOptions, +): void { + if (!customProviders) return; + for (const provider of customProviders.providers) { + modelRegistry.registerProvider(provider.name, { + baseUrl: provider.baseUrl, + api: provider.api, + ...(provider.headers ? { headers: provider.headers } : {}), + ...(provider.apiKey ? { apiKey: provider.apiKey } : {}), + models: provider.models, + }); + if (provider.apiKey) { + authStorage.setRuntimeApiKey(provider.name, provider.apiKey); + } + } +} + function resolvePiModel( model: string | undefined, registry: ModelRegistry, @@ -345,6 +368,7 @@ async function runPiPrompt(options: PiPromptOptions): Promise { bridgeWardenProviderApiKeyEnv(); const authStorage = createAuthStorage(options.model, options.legacyAnthropicApiKey); const modelRegistry = ModelRegistry.create(authStorage); + registerCustomProviders(modelRegistry, authStorage, options.customProviders); const model = resolvePiModel(options.model, modelRegistry); const settingsManager = buildSettingsManager(options.timeout, options.maxRetries); const agentDir = getAgentDir(); @@ -655,6 +679,7 @@ async function runStructured( tools?: AuxiliaryTool[]; executeTool?: (name: string, input: Record) => Promise; maxIterations?: number; + providerOptions?: unknown; } ): Promise> { const customTools = toPiCustomTools(request.tools, request.executeTool); @@ -687,6 +712,7 @@ async function runStructured( agentName: request.agentName, model: request.model, legacyAnthropicApiKey: request.apiKey, + customProviders: request.providerOptions as PiProviderOptions, toolNames, customTools, toolDescriptions, @@ -758,6 +784,7 @@ export const piRuntime: Runtime = { skillName, tools, allowMutatingTools, + providerOptions, } = request; const { maxTurns = 50, model, effort, abortController } = options; const skillTools = resolvePiSkillTools(tools, allowMutatingTools); @@ -787,6 +814,7 @@ export const piRuntime: Runtime = { agentName: skillName, model, legacyAnthropicApiKey: apiKey, + customProviders: providerOptions as PiProviderOptions, toolNames: skillTools.toolNames, maxTurns, effort, From 09aff53b6ed4f973998d094a4bd3cb29dc6e7f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Sat, 27 Jun 2026 00:24:36 +0200 Subject: [PATCH 09/22] feat(runtime): thread custom providers through all model lanes - Add providers?: ProvidersConfig to SkillRunnerOptions, AuxiliaryCallOptions, VerifyFindingsOptions, PostProcessFindingsOptions, JsonOutputRepairOptions, and FixJudgeRuntimeOptions - Carry providers through every child-options construction site in analyze.ts and post-process.ts so verify/dedup lanes do not silently lose the field - Attach providerOptions: getRuntimeProviderOptions(..., { providers }) to every runAuxiliary/runSynthesis call in extract.ts, dedup.ts, json-output.ts, judge.ts, and verify.ts - Extend findSemanticDuplicates Pick type to include providers - Add focused integration test asserting providerOptions is forwarded - Fix judge.runtime-options.test.ts mock to include getRuntimeProviderOptions Co-Authored-By: Claude Opus 4.8 --- .../judge.runtime-options.test.ts | 1 + .../warden/src/action/fix-evaluation/judge.ts | 6 +- packages/warden/src/output/dedup.ts | 6 +- packages/warden/src/sdk/analyze.ts | 3 + packages/warden/src/sdk/extract.test.ts | 150 +++++++++++------- packages/warden/src/sdk/extract.ts | 7 +- packages/warden/src/sdk/json-output.ts | 6 +- packages/warden/src/sdk/post-process.ts | 6 +- packages/warden/src/sdk/types.ts | 4 +- packages/warden/src/sdk/verify.ts | 5 +- 10 files changed, 125 insertions(+), 69 deletions(-) diff --git a/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts b/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts index aef185ac..a2920217 100644 --- a/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts +++ b/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts @@ -19,6 +19,7 @@ describe('evaluateFix runtime options', () => { vi.doMock('../../sdk/runtimes/index.js', () => ({ getRuntime, + getRuntimeProviderOptions: vi.fn(() => undefined), })); const { evaluateFix } = await import('./judge.js'); diff --git a/packages/warden/src/action/fix-evaluation/judge.ts b/packages/warden/src/action/fix-evaluation/judge.ts index 1b069d7f..d9372c6c 100644 --- a/packages/warden/src/action/fix-evaluation/judge.ts +++ b/packages/warden/src/action/fix-evaluation/judge.ts @@ -7,7 +7,8 @@ import { buildTaggedSection, joinPromptSections, } from '../../sdk/prompt-sections.js'; -import { getRuntime, type AuxiliaryTool, type RuntimeName } from '../../sdk/runtimes/index.js'; +import { getRuntime, getRuntimeProviderOptions, type AuxiliaryTool, type RuntimeName } from '../../sdk/runtimes/index.js'; +import type { ProvidersConfig } from '../../config/schema.js'; import { emptyUsage } from '../../sdk/usage.js'; import { FixJudgeVerdictSchema } from './types.js'; import type { FixJudgeResult } from './types.js'; @@ -33,6 +34,8 @@ export interface FixJudgeContext { export interface FixJudgeRuntimeOptions { runtime?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; model?: string; maxRetries?: number; } @@ -234,6 +237,7 @@ export async function evaluateFix( model: runtimeOptions.model, maxIterations: 5, maxRetries: runtimeOptions.maxRetries, + providerOptions: getRuntimeProviderOptions(runtimeOptions.runtime ?? 'claude', { providers: runtimeOptions.providers }), }); if (result.success) { diff --git a/packages/warden/src/output/dedup.ts b/packages/warden/src/output/dedup.ts index eca21645..de0b168b 100644 --- a/packages/warden/src/output/dedup.ts +++ b/packages/warden/src/output/dedup.ts @@ -3,7 +3,7 @@ import type { Octokit } from '@octokit/rest'; import { z } from 'zod'; import type { Confidence, Finding, Severity, UsageStats } from '../types/index.js'; import { findingLine } from '../types/index.js'; -import { getRuntime } from '../sdk/runtimes/index.js'; +import { getRuntime, getRuntimeProviderOptions } from '../sdk/runtimes/index.js'; import { applyMergeGroups, canUseRuntimeAuth } from '../sdk/extract.js'; import type { AuxiliaryCallOptions } from '../sdk/extract.js'; import { @@ -581,7 +581,7 @@ async function findSemanticDuplicates( findings: Finding[], existingComments: ExistingComment[], apiKey: string | undefined, - options: Pick = {} + options: Pick = {} ): Promise { if (findings.length === 0 || existingComments.length === 0) { return { matches: new Map() }; @@ -622,6 +622,7 @@ Return [] if none are duplicates.`), model: options.model, maxTokens: 512, maxRetries: options.maxRetries, + providerOptions: getRuntimeProviderOptions(options.runtime ?? 'claude', { providers: options.providers }), }); if (!result.success) { @@ -925,6 +926,7 @@ Singletons (findings with no duplicates) should not appear in any group. model: options.model, maxTokens: 512, maxRetries: options.maxRetries, + providerOptions: getRuntimeProviderOptions(options.runtime ?? 'claude', { providers: options.providers }), }); if (!result.success) { diff --git a/packages/warden/src/sdk/analyze.ts b/packages/warden/src/sdk/analyze.ts index 14bcc5a1..265dfe29 100644 --- a/packages/warden/src/sdk/analyze.ts +++ b/packages/warden/src/sdk/analyze.ts @@ -164,6 +164,7 @@ async function parseHunkOutput( const fallback = await extractFindingsWithLLM(result.text, { apiKey: options.apiKey, runtime: options.runtime, + providers: options.providers, model: options.auxiliaryModel, maxRetries: options.auxiliaryMaxRetries, agentName: skillName, @@ -396,6 +397,7 @@ async function analyzeHunk( }, providerOptions: getRuntimeProviderOptions(runtimeName, { pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable, + providers: options.providers, }), })); @@ -1159,6 +1161,7 @@ async function runSkillAnalysis( repoPath: context.repoPath, apiKey: options.apiKey, runtime: options.runtime, + providers: options.providers, auxiliaryModel: options.auxiliaryModel, synthesisModel: options.synthesisModel, auxiliaryMaxRetries: options.auxiliaryMaxRetries, diff --git a/packages/warden/src/sdk/extract.test.ts b/packages/warden/src/sdk/extract.test.ts index 5c9bbdc6..11c9d69e 100644 --- a/packages/warden/src/sdk/extract.test.ts +++ b/packages/warden/src/sdk/extract.test.ts @@ -10,18 +10,43 @@ import { canUseRuntimeAuth, } from './extract.js'; import type { Finding } from '../types/index.js'; +import { getRuntime } from './runtimes/index.js'; +import type { Runtime } from './runtimes/index.js'; + +// Mock runtimes to avoid real API calls +vi.mock('./runtimes/index.js', () => ({ + getRuntime: vi.fn(), + getRuntimeProviderOptions: vi.fn((_name: string, opts: { providers?: unknown }) => { + if (!opts?.providers) return undefined; + // Minimal simulation: return providerOptions shape used by Pi adapter + const providers = opts.providers as Record; + return { + providers: Object.entries(providers).map(([name]) => ({ name })), + }; + }), +})); -// Mock callHaiku to avoid real API calls -vi.mock('./haiku.js', async (importOriginal) => { - const actual: Record = await importOriginal(); +function makeSynthesisRuntime(data: unknown): Runtime { return { - ...actual, - callHaiku: vi.fn(), - }; -}); + name: 'claude' as const, + runSkill: vi.fn(), + runAuxiliary: vi.fn(), + runSynthesis: vi.fn().mockResolvedValue({ + success: true, + data, + usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, + }), + } as unknown as Runtime; +} -import { callHaiku } from './haiku.js'; -const mockCallHaiku = vi.mocked(callHaiku); +function makeAuxiliaryRuntime(result: unknown): Runtime { + return { + name: 'pi' as const, + runSkill: vi.fn(), + runAuxiliary: vi.fn().mockResolvedValue(result), + runSynthesis: vi.fn(), + } as unknown as Runtime; +} function makeFinding(overrides: Partial = {}): Finding { return { @@ -45,11 +70,12 @@ describe('extractFindingsWithLLM', () => { }); it('preserves the LLM extraction failure prefix for stable error classification', async () => { - mockCallHaiku.mockResolvedValue({ + const runtime = makeAuxiliaryRuntime({ success: false, error: 'Request timed out', usage: { inputTokens: 10, outputTokens: 0, costUSD: 0.001 }, }); + vi.mocked(getRuntime).mockReturnValue(runtime); const result = await extractFindingsWithLLM('{ "findings": [', { apiKey: 'test-key' }); @@ -60,6 +86,38 @@ describe('extractFindingsWithLLM', () => { usage: { inputTokens: 10, outputTokens: 0, costUSD: 0.001 }, }); }); + + it('forwards providerOptions built from providers to the auxiliary runtime', async () => { + const runAuxiliaryMock = vi.fn().mockResolvedValue({ + success: true, + data: { findings: [] }, + usage: { inputTokens: 10, outputTokens: 5, costUSD: 0.001 }, + }); + const runtime: Runtime = { + name: 'pi', + runSkill: vi.fn(), + runAuxiliary: runAuxiliaryMock, + runSynthesis: vi.fn(), + } as unknown as Runtime; + vi.mocked(getRuntime).mockReturnValue(runtime); + + const rawText = '{ "findings": [] }'; + await extractFindingsWithLLM(rawText, { + runtime: 'pi', + providers: { + litellm: { + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions', + models: [{ id: 'm' }], + }, + }, + }); + + const req = runAuxiliaryMock.mock.calls[0]?.[0]; + expect(req).toBeDefined(); + expect(req.providerOptions).toBeDefined(); + expect(req.providerOptions.providers[0].name).toBe('litellm'); + }); }); describe('deduplicateFindings', () => { @@ -134,11 +192,8 @@ describe('mergeCrossLocationFindings', () => { makeFinding({ id: 'f2', title: 'Issue B', location: { path: 'src/b.ts', startLine: 1 } }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + const runtime = makeSynthesisRuntime([]); + vi.mocked(getRuntime).mockReturnValue(runtime); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', @@ -147,7 +202,7 @@ describe('mergeCrossLocationFindings', () => { }); expect(result.findings).toHaveLength(2); expect(result.mergedCount).toBe(0); - expect(mockCallHaiku).toHaveBeenCalledWith(expect.objectContaining({ + expect(vi.mocked(runtime.runSynthesis)).toHaveBeenCalledWith(expect.objectContaining({ model: 'claude-test-fast', })); }); @@ -168,11 +223,7 @@ describe('mergeCrossLocationFindings', () => { }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); const onFindingProcessing = vi.fn(); const result = await mergeCrossLocationFindings(findings, { @@ -203,11 +254,7 @@ describe('mergeCrossLocationFindings', () => { makeFinding({ id: 'f3', severity: 'medium', location: { path: 'src/a.ts', startLine: 5 } }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2, 3]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2, 3]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); expect(result.findings).toHaveLength(1); @@ -233,11 +280,7 @@ describe('mergeCrossLocationFindings', () => { }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); // Same severity, same confidence, a.ts < b.ts alphabetically @@ -259,11 +302,7 @@ describe('mergeCrossLocationFindings', () => { }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); expect(result.findings[0]!.additionalLocations).toEqual([ @@ -279,11 +318,7 @@ describe('mergeCrossLocationFindings', () => { makeFinding({ id: 'f3', location: { path: 'src/b.ts', startLine: 2 } }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], // groups the two with-location findings (indices 1,2 from withLocations array) - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); // f1 absorbs f3, f2 (no location) passes through @@ -297,11 +332,16 @@ describe('mergeCrossLocationFindings', () => { makeFinding({ id: 'f2', location: { path: 'src/b.ts', startLine: 2 } }), ]; - mockCallHaiku.mockResolvedValue({ - success: false, - error: 'API error', - usage: { inputTokens: 100, outputTokens: 0, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue({ + name: 'claude', + runSkill: vi.fn(), + runAuxiliary: vi.fn(), + runSynthesis: vi.fn().mockResolvedValue({ + success: false, + error: 'API error', + usage: { inputTokens: 100, outputTokens: 0, costUSD: 0.001 }, + }), + } as unknown as Runtime); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); expect(result.findings).toHaveLength(2); @@ -323,11 +363,7 @@ describe('mergeCrossLocationFindings', () => { location: { path: 'src/b.ts', startLine: 2 }, }); - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); await mergeCrossLocationFindings([f1, f2], { apiKey: 'test-key', repoPath: tempDir }); // Original f1 should NOT have additionalLocations added @@ -342,11 +378,7 @@ describe('mergeCrossLocationFindings', () => { ]; // LLM returns overlapping groups: [1,2] and [2,3] - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2], [2, 3]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2], [2, 3]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); // f2 is absorbed by group [1,2]. Group [2,3] should skip f2 (already absorbed), @@ -372,11 +404,7 @@ describe('mergeCrossLocationFindings', () => { }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); // src/b.ts:2 should only appear once in additionalLocations (deduped) diff --git a/packages/warden/src/sdk/extract.ts b/packages/warden/src/sdk/extract.ts index b7ef0229..ac9d00b9 100644 --- a/packages/warden/src/sdk/extract.ts +++ b/packages/warden/src/sdk/extract.ts @@ -4,8 +4,9 @@ import { z } from 'zod'; import { customAlphabet } from 'nanoid'; import { FindingSchema, compareFindingPriority } from '../types/index.js'; import type { Finding, Location, UsageStats } from '../types/index.js'; -import { getRuntime } from './runtimes/index.js'; +import { getRuntime, getRuntimeProviderOptions } from './runtimes/index.js'; import type { RuntimeName } from './runtimes/index.js'; +import type { ProvidersConfig } from '../config/schema.js'; import type { FindingProcessingEvent } from './types.js'; import { buildJsonOutputSection, @@ -29,6 +30,8 @@ export type ExtractFindingsResult = export interface AuxiliaryCallOptions { apiKey?: string; runtime?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; model?: string; maxRetries?: number; agentName?: string; @@ -237,6 +240,7 @@ If no findings exist, return: {"findings": []}`), maxTokens: LLM_FALLBACK_MAX_TOKENS, timeout: LLM_FALLBACK_TIMEOUT_MS, maxRetries: options.maxRetries, + providerOptions: getRuntimeProviderOptions(runtimeName, { providers: options.providers }), }); if (!result.success) { @@ -556,6 +560,7 @@ Singletons should not appear. Return [] if no findings describe the same issue.` model: options?.model, maxTokens: 512, maxRetries: options?.maxRetries, + providerOptions: getRuntimeProviderOptions(options?.runtime ?? 'claude', { providers: options?.providers }), }); if (!result.success) { diff --git a/packages/warden/src/sdk/json-output.ts b/packages/warden/src/sdk/json-output.ts index e0279644..01c84300 100644 --- a/packages/warden/src/sdk/json-output.ts +++ b/packages/warden/src/sdk/json-output.ts @@ -3,7 +3,8 @@ import type { UsageStats } from '../types/index.js'; import { extractJson } from './haiku.js'; import { canUseRuntimeAuth } from './extract.js'; import { buildJsonOutputSection, buildTaggedSection, joinPromptSections } from './prompt-sections.js'; -import { getRuntime, type Runtime, type RuntimeName } from './runtimes/index.js'; +import { getRuntime, getRuntimeProviderOptions, type Runtime, type RuntimeName } from './runtimes/index.js'; +import type { ProvidersConfig } from '../config/schema.js'; const JSON_REPAIR_MAX_CHARS = 60_000; const JSON_REPAIR_MAX_TOKENS = 16_384; @@ -18,6 +19,8 @@ export interface JsonOutputRepairOptions { agentName?: string; runtime?: Runtime; runtimeName?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; model?: string; maxRetries?: number; maxTokens?: number; @@ -86,6 +89,7 @@ async function repairJsonOutput( maxTokens: repair.maxTokens ?? JSON_REPAIR_MAX_TOKENS, timeout: repair.timeout ?? JSON_REPAIR_TIMEOUT_MS, schema, + providerOptions: getRuntimeProviderOptions(runtime.name, { providers: repair.providers }), prompt: joinPromptSections([ ` Extract and repair the JSON value from this model output. diff --git a/packages/warden/src/sdk/post-process.ts b/packages/warden/src/sdk/post-process.ts index 1660ab93..d670d4c2 100644 --- a/packages/warden/src/sdk/post-process.ts +++ b/packages/warden/src/sdk/post-process.ts @@ -1,4 +1,4 @@ -import type { Effort, SkillDefinition } from '../config/schema.js'; +import type { Effort, ProvidersConfig, SkillDefinition } from '../config/schema.js'; import { emitDedupMetrics } from '../sentry.js'; import type { Finding } from '../types/index.js'; import { deduplicateFindings, mergeCrossLocationFindings } from './extract.js'; @@ -12,6 +12,8 @@ export interface PostProcessFindingsOptions { repoPath: string; apiKey?: string; runtime?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; auxiliaryModel?: string; synthesisModel?: string; auxiliaryMaxRetries?: number; @@ -48,6 +50,7 @@ export async function postProcessFindings( skill: options.skill, apiKey: options.apiKey, runtime: options.runtime, + providers: options.providers, model: options.auxiliaryModel, maxTurns: options.maxTurns, effort: options.effort, @@ -71,6 +74,7 @@ export async function postProcessFindings( apiKey: options.apiKey, repoPath: options.repoPath, runtime: options.runtime, + providers: options.providers, model: options.synthesisModel, maxRetries: options.auxiliaryMaxRetries, agentName: options.skill.name, diff --git a/packages/warden/src/sdk/types.ts b/packages/warden/src/sdk/types.ts index 7c773562..206fc7b7 100644 --- a/packages/warden/src/sdk/types.ts +++ b/packages/warden/src/sdk/types.ts @@ -1,6 +1,6 @@ import type { Finding, UsageStats, SkippedFile, RetryConfig, ErrorCode, HunkFailure, HunkTrace } from '../types/index.js'; import type { HunkWithContext } from '../diff/index.js'; -import type { ChunkingConfig, Effort, IgnoreConfig, ScanConfig } from '../config/schema.js'; +import type { ChunkingConfig, Effort, IgnoreConfig, ProvidersConfig, ScanConfig } from '../config/schema.js'; import type { RuntimeName } from './runtimes/index.js'; import type { ProviderFailureCircuitBreaker } from './circuit-breaker.js'; @@ -114,6 +114,8 @@ export interface SkillRunnerOptions { effort?: Effort; /** Runtime backend for all model-backed execution. Defaults to Pi. */ runtime?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; /** Model to use for auxiliary structured model calls. Uses runtime default if not specified. */ auxiliaryModel?: string; /** Model to use for post-analysis synthesis/consolidation. Falls back to auxiliaryModel when not specified. */ diff --git a/packages/warden/src/sdk/verify.ts b/packages/warden/src/sdk/verify.ts index 440ead65..43f5f18c 100644 --- a/packages/warden/src/sdk/verify.ts +++ b/packages/warden/src/sdk/verify.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { Effort, SkillDefinition } from '../config/schema.js'; +import type { Effort, ProvidersConfig, SkillDefinition } from '../config/schema.js'; import { FindingSchema, type Finding, type UsageStats } from '../types/index.js'; import { aggregateUsage } from './usage.js'; import { extractBalancedJson } from './extract.js'; @@ -32,6 +32,8 @@ export interface VerifyFindingsOptions { skill: SkillDefinition; apiKey?: string; runtime?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; model?: string; maxTurns?: number; effort?: Effort; @@ -267,6 +269,7 @@ export async function verifyFindings( tools: options.skill.tools, providerOptions: getRuntimeProviderOptions(runtimeName, { pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable, + providers: options.providers, }), }); From c43577611e9b37384a9af76bb1f632e5695e6cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Sat, 27 Jun 2026 00:36:23 +0200 Subject: [PATCH 10/22] feat(config): propagate custom providers to all runner entry points Co-Authored-By: Claude Opus 4.8 --- packages/warden/src/action/review/poster.ts | 4 ++++ packages/warden/src/action/triggers/executor.ts | 1 + packages/warden/src/action/workflow/pr-workflow.ts | 4 ++++ packages/warden/src/action/workflow/schedule.ts | 1 + packages/warden/src/cli/main.ts | 6 ++++++ packages/warden/src/config/loader.test.ts | 13 +++++++++++++ packages/warden/src/config/loader.ts | 6 ++++++ 7 files changed, 35 insertions(+) diff --git a/packages/warden/src/action/review/poster.ts b/packages/warden/src/action/review/poster.ts index 29d00774..e2a1e048 100644 --- a/packages/warden/src/action/review/poster.ts +++ b/packages/warden/src/action/review/poster.ts @@ -21,6 +21,7 @@ import type { ExistingComment, DeduplicateResult } from '../../output/dedup.js'; import { mergeAuxiliaryUsage, mergeAuxiliaryUsageAttribution } from '../../sdk/usage.js'; import { canUseRuntimeAuth } from '../../sdk/extract.js'; import type { RuntimeName } from '../../sdk/runtimes/index.js'; +import type { ProvidersConfig } from '../../config/schema.js'; import type { TriggerResult } from '../triggers/executor.js'; import { logAction, warnAction } from '../../cli/output/tty.js'; import type { FindingObservation } from '../reporting/outcomes.js'; @@ -37,6 +38,7 @@ export interface ReviewPostingContext { existingComments: ExistingComment[]; apiKey: string; runtime?: RuntimeName; + providers?: ProvidersConfig; model?: string; maxRetries?: number; /** Throw review posting failures instead of converting them to warnings. */ @@ -258,6 +260,7 @@ export async function postTriggerReview( const consolidateResult = await consolidateBatchFindings(findingsToPost, { apiKey, runtime: ctx.runtime, + providers: ctx.providers, model: ctx.model, hashOnly: !canUseAuxiliaryRuntime, maxRetries: ctx.maxRetries, @@ -297,6 +300,7 @@ export async function postTriggerReview( dedupResult = await deduplicateFindings(findingsToPost, existingComments, { apiKey, runtime: ctx.runtime, + providers: ctx.providers, model: ctx.model, currentSkill: skill, maxRetries: ctx.maxRetries, diff --git a/packages/warden/src/action/triggers/executor.ts b/packages/warden/src/action/triggers/executor.ts index 90affb87..2b4360bb 100644 --- a/packages/warden/src/action/triggers/executor.ts +++ b/packages/warden/src/action/triggers/executor.ts @@ -187,6 +187,7 @@ export async function executeTrigger( apiKey: anthropicApiKey, model: trigger.model, runtime: trigger.runtime, + providers: trigger.providers, effort: trigger.effort, auxiliaryModel: trigger.auxiliaryModel, synthesisModel: trigger.synthesisModel, diff --git a/packages/warden/src/action/workflow/pr-workflow.ts b/packages/warden/src/action/workflow/pr-workflow.ts index 7546b383..27910844 100644 --- a/packages/warden/src/action/workflow/pr-workflow.ts +++ b/packages/warden/src/action/workflow/pr-workflow.ts @@ -45,6 +45,7 @@ import { postTriggerReview } from '../review/poster.js'; import { shouldResolveStaleComments } from '../review/coordination.js'; import type { FindingObservation } from '../reporting/outcomes.js'; import type { RuntimeName } from '../../sdk/runtimes/index.js'; +import type { ProvidersConfig } from '../../config/schema.js'; import { canUseRuntimeAuth } from '../../sdk/extract.js'; import { ProviderFailureCircuitBreaker } from '../../sdk/circuit-breaker.js'; import { @@ -119,6 +120,7 @@ interface FixEvaluationCommentGroups { interface AuxiliaryWorkflowOptions { runtime?: RuntimeName; + providers?: ProvidersConfig; model?: string; maxRetries?: number; } @@ -182,6 +184,7 @@ function resolveWorkflowAuxiliaryOptions(layered: LoadedLayeredConfig): Auxiliar // trigger, so the org base config remains the enforced baseline and the // repo layer only fills fields the base omits. runtime: baseDefaults?.runtime ?? repoDefaults?.runtime ?? 'pi', + providers: baseDefaults?.providers ?? repoDefaults?.providers, model: emptyToUndefined(baseDefaults?.auxiliary?.model) ?? emptyToUndefined(repoDefaults?.auxiliary?.model), @@ -564,6 +567,7 @@ async function postReviewsAndTrackFailures( existingComments, apiKey: inputs.anthropicApiKey, runtime: auxiliaryOptions.runtime, + providers: auxiliaryOptions.providers, model: auxiliaryOptions.model, maxRetries: auxiliaryOptions.maxRetries, failOnPostError: options.failOnPostError, diff --git a/packages/warden/src/action/workflow/schedule.ts b/packages/warden/src/action/workflow/schedule.ts index ba3d36ea..c4488884 100644 --- a/packages/warden/src/action/workflow/schedule.ts +++ b/packages/warden/src/action/workflow/schedule.ts @@ -214,6 +214,7 @@ async function runScheduleWorkflowInner( apiKey: inputs.anthropicApiKey, model: resolved.model, runtime: resolved.runtime, + providers: resolved.providers, effort: resolved.effort, auxiliaryModel: resolved.auxiliaryModel, synthesisModel: resolved.synthesisModel, diff --git a/packages/warden/src/cli/main.ts b/packages/warden/src/cli/main.ts index e3f3135a..ae3e7ef7 100644 --- a/packages/warden/src/cli/main.ts +++ b/packages/warden/src/cli/main.ts @@ -612,6 +612,7 @@ interface SkillToRun { maxTurns?: number; effort?: SkillRunnerOptions['effort']; runtime?: SkillRunnerOptions['runtime']; + providers?: SkillRunnerOptions['providers']; auxiliaryModel?: string; synthesisModel?: string; auxiliaryMaxRetries?: number; @@ -643,6 +644,7 @@ type SkillRunnerOptionOverrides = Pick< | 'maxTurns' | 'effort' | 'runtime' + | 'providers' | 'auxiliaryModel' | 'synthesisModel' | 'auxiliaryMaxRetries' @@ -709,6 +711,7 @@ export function mergeSkillRunnerOptions( if (overrides.maxTurns !== undefined) merged.maxTurns = overrides.maxTurns; if (overrides.effort !== undefined) merged.effort = overrides.effort; if (overrides.runtime !== undefined) merged.runtime = overrides.runtime; + if (overrides.providers !== undefined) merged.providers = overrides.providers; if (overrides.auxiliaryModel !== undefined) merged.auxiliaryModel = overrides.auxiliaryModel; if (overrides.synthesisModel !== undefined) merged.synthesisModel = overrides.synthesisModel; if (overrides.auxiliaryMaxRetries !== undefined) { @@ -1098,6 +1101,7 @@ export async function runSkills( maxTurns: match?.maxTurns ?? config?.defaults?.agent?.maxTurns ?? config?.defaults?.maxTurns, effort: options.effort ?? match?.effort ?? defaultEffort, runtime: options.runtime ?? match?.runtime ?? config?.defaults?.runtime ?? 'pi', + providers: match?.providers ?? config?.defaults?.providers, auxiliaryModel: match?.auxiliaryModel ?? defaultAuxiliaryModel, synthesisModel: match?.synthesisModel ?? defaultSynthesisModel, auxiliaryMaxRetries: @@ -1125,6 +1129,7 @@ export async function runSkills( model: t.model, maxTurns: t.maxTurns, runtime: options.runtime ?? t.runtime, + providers: t.providers, effort: options.effort ?? t.effort, auxiliaryModel: t.auxiliaryModel, synthesisModel: t.synthesisModel, @@ -1515,6 +1520,7 @@ async function runConfigMode(options: CLIOptions, reporter: Reporter): Promise { expect(resolved?.minConfidence).toBeUndefined(); }); }); + + it('propagates defaults.providers onto every resolved trigger', () => { + const config = WardenConfigSchema.parse({ + version: 1, + defaults: { providers: { litellm: { baseUrl: 'http://localhost:4000/v1', models: [{ id: 'm' }] } } }, + skills: [{ name: 'a' }, { name: 'b', triggers: [{ type: 'local' }] }], + }); + const resolved = resolveSkillConfigs(config); + expect(resolved).toHaveLength(2); + for (const trigger of resolved) { + expect(trigger.providers?.['litellm']?.baseUrl).toBe('http://localhost:4000/v1'); + } + }); }); describe('mergeWardenConfigs', () => { diff --git a/packages/warden/src/config/loader.ts b/packages/warden/src/config/loader.ts index 20e43d5a..bc36140a 100644 --- a/packages/warden/src/config/loader.ts +++ b/packages/warden/src/config/loader.ts @@ -19,6 +19,7 @@ import { type LogsConfig, type RuntimeName, type AgentRuntimeConfig, + type ProvidersConfig, } from './schema.js'; import type { SeverityThreshold, ConfidenceThreshold } from '../types/index.js'; @@ -399,6 +400,8 @@ export interface ResolvedTrigger { effort?: AgentRuntimeConfig['effort']; /** Runtime backend for all model-backed execution. */ runtime?: RuntimeName; + /** Custom OpenAI-compatible providers for the Pi runtime. */ + providers?: ProvidersConfig; /** Model for auxiliary structured model calls. */ auxiliaryModel?: string; /** Model for post-analysis synthesis/consolidation. */ @@ -493,6 +496,7 @@ export function resolveSkillConfigs( const envModel = emptyToUndefined(process.env['WARDEN_MODEL']); const result: ResolvedTrigger[] = []; const runtime = defaults?.runtime ?? 'pi'; + const providers = defaults?.providers; const auxiliaryModel = emptyToUndefined(defaults?.auxiliary?.model); const synthesisModel = emptyToUndefined(defaults?.synthesis?.model) ?? @@ -544,6 +548,7 @@ export function resolveSkillConfigs( maxTurns: baseMaxTurns, effort, runtime, + providers, auxiliaryModel, synthesisModel, auxiliaryMaxRetries, @@ -579,6 +584,7 @@ export function resolveSkillConfigs( maxTurns: trigger.maxTurns ?? baseMaxTurns, effort, runtime, + providers, auxiliaryModel, synthesisModel, auxiliaryMaxRetries, From e1055560a2cb98c91aff446a588967991a5d4107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Sat, 27 Jun 2026 00:47:20 +0200 Subject: [PATCH 11/22] feat(skill-builder): route build/improve through custom providers Co-Authored-By: Claude Opus 4.8 --- packages/warden/src/cli/commands/build.ts | 3 + packages/warden/src/skill-builder/agentic.ts | 8 +- .../warden/src/skill-builder/outline.test.ts | 94 ++++++++++++++++++- packages/warden/src/skill-builder/outline.ts | 7 +- packages/warden/src/skill-builder/skill.ts | 6 ++ 5 files changed, 113 insertions(+), 5 deletions(-) diff --git a/packages/warden/src/cli/commands/build.ts b/packages/warden/src/cli/commands/build.ts index 48a0f38b..45d2bbd1 100644 --- a/packages/warden/src/cli/commands/build.ts +++ b/packages/warden/src/cli/commands/build.ts @@ -331,6 +331,7 @@ async function runGeneratedSkillCommand( : collectSkillBuildSource(skill); const runtimeName = config?.defaults?.runtime ?? 'pi'; + const providers = config?.defaults?.providers; const model = resolveSynthesisModel(config, options); const repairModel = emptyToUndefined(config?.defaults?.auxiliary?.model); const maxRetries = config?.defaults?.auxiliary?.maxRetries ?? config?.defaults?.auxiliaryMaxRetries; @@ -387,6 +388,7 @@ async function runGeneratedSkillCommand( source, repairModel, repairMaxRetries: maxRetries, + providers, onStatus: setDetail, }), }); @@ -435,6 +437,7 @@ async function runGeneratedSkillCommand( repairMaxRetries: maxRetries, abortController: state?.abortController, regenerate: options.regenerate || outlineResult.source === 'generated' || mode === 'improve', + providers, onStatus: setDetail, }), }); diff --git a/packages/warden/src/skill-builder/agentic.ts b/packages/warden/src/skill-builder/agentic.ts index ad3b9dff..448a6e90 100644 --- a/packages/warden/src/skill-builder/agentic.ts +++ b/packages/warden/src/skill-builder/agentic.ts @@ -1,11 +1,12 @@ import { performance } from 'node:perf_hooks'; import { z } from 'zod'; -import type { ToolName } from '../config/schema.js'; +import type { ProvidersConfig, ToolName } from '../config/schema.js'; import type { UsageStats } from '../types/index.js'; import { parseJsonFromOutput, type ParseJsonFromOutputResult } from '../sdk/json-output.js'; import { buildJsonOutputSection, buildTaggedSection, joinPromptSections } from '../sdk/prompt-sections.js'; import { aggregateUsage, emptyUsage } from '../sdk/usage.js'; import type { Runtime, SkillRunResult } from '../sdk/runtimes/index.js'; +import { getRuntimeProviderOptions } from '../sdk/runtimes/index.js'; const SKILL_BUILDER_READ_TOOLS: ToolName[] = ['Read', 'Grep', 'Glob', 'WebFetch', 'WebSearch']; const SKILL_BUILDER_WRITE_TOOLS: ToolName[] = ['Read', 'Grep', 'Glob', 'Write', 'Edit', 'Bash', 'WebFetch', 'WebSearch']; @@ -135,6 +136,7 @@ async function repairStructuredSkillBuilderOutput(args: { reason: string; model?: string; abortController?: AbortController; + providers?: ProvidersConfig; }): Promise> { const response = await args.runtime.runSkill({ apiKey: args.apiKey, @@ -152,6 +154,7 @@ async function repairStructuredSkillBuilderOutput(args: { maxTurns: STRUCTURED_REPAIR_MAX_TURNS, abortController: args.abortController, }, + providerOptions: getRuntimeProviderOptions(args.runtime.name, { providers: args.providers }), }); if (response.authError) { @@ -208,6 +211,7 @@ export async function runStructuredSkillBuilderAgent(args: { writeAccess?: boolean; abortController?: AbortController; apiKey?: string; + providers?: ProvidersConfig; repair?: { apiKey?: string; model?: string; @@ -229,6 +233,7 @@ export async function runStructuredSkillBuilderAgent(args: { maxTurns: args.maxTurns, abortController: args.abortController, }, + providerOptions: getRuntimeProviderOptions(runtime.name, { providers: args.providers }), }); if (response.authError) { @@ -263,6 +268,7 @@ export async function runStructuredSkillBuilderAgent(args: { reason: parsed.error, model: args.repair?.model ?? args.model, abortController: args.abortController, + providers: args.providers, }); if (skillRepair.usage) { repairUsages.push(skillRepair.usage); diff --git a/packages/warden/src/skill-builder/outline.test.ts b/packages/warden/src/skill-builder/outline.test.ts index e7693984..121584a9 100644 --- a/packages/warden/src/skill-builder/outline.test.ts +++ b/packages/warden/src/skill-builder/outline.test.ts @@ -1,8 +1,11 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, describe, expect, it } from 'vitest'; -import { collectSkillImproveSource } from './outline.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { Runtime, SynthesisRunRequest } from '../sdk/runtimes/index.js'; +import type { ProvidersConfig } from '../config/schema.js'; +import { buildSkillOutline, collectSkillImproveSource } from './outline.js'; +import { SKILL_BUILD_VERSION, SKILL_BUILD_OUTLINE_SCHEMA_VERSION, type SkillBuildOutline } from './outline-contract.js'; import { getBuildStatePath, readSkillBuildState, @@ -161,3 +164,90 @@ prompt: Find security issues. ); }); }); + +describe('buildSkillOutline: custom providers forwarding', () => { + const tempDirs: string[] = []; + + afterEach(() => { + for (const dir of tempDirs) { + rmSync(dir, { recursive: true, force: true }); + } + tempDirs.length = 0; + }); + + it('forwards providerOptions to runSynthesis when providers are configured', async () => { + const rootDir = mkdtempSync(join(tmpdir(), 'warden-outline-providers-')); + tempDirs.push(rootDir); + // Write a minimal warden.yaml so generatedSkillDefinitionSourceFile can load it. + writeFileSync(join(rootDir, 'warden.yaml'), `version: 1\nkind: generated-skill\nname: security\nprompt: Find security issues.\n`, 'utf-8'); + + const providers: ProvidersConfig = { + litellm: { + baseUrl: 'http://localhost:4000', + api: 'openai-completions', + models: [{ id: 'litellm/gpt-4o', name: 'gpt-4o' }], + }, + }; + + // Build a minimal valid outline to return from the stub. + const fakeOutline: SkillBuildOutline = { + version: SKILL_BUILD_OUTLINE_SCHEMA_VERSION, + skill: 'security', + sourceHash: 'placeholder', // overwritten after source is collected + buildVersion: SKILL_BUILD_VERSION, + scopeProfile: { + kind: 'domain', + subject: 'Generic security review', + localContextUsed: false, + observedContext: ['Generic security review'], + unresolvedContext: [], + }, + build: { + phases: [{ id: 'build-tracks', status: 'generated' }], + }, + tracks: [{ + id: 'auth-bypass', + title: 'Authentication bypasses', + goal: 'Find broken authentication checks.', + rationale: 'Authentication bugs are core security issues.', + sourceSignals: ['Auth endpoints'], + owns: ['Missing auth checks'], + excludes: [], + relevanceSignals: ['Session checks'], + evidenceFocus: ['Changed auth conditions'], + checks: ['Trace auth preconditions'], + safeCounterpatterns: ['Explicit user verification'], + falsePositiveTraps: ['Defense-in-depth logging'], + researchHints: [], + }], + }; + + const capturedRequests: SynthesisRunRequest[] = []; + + const mockRuntime: Runtime = { + name: 'pi', + runSkill: vi.fn().mockResolvedValue({ result: undefined }), + runAuxiliary: vi.fn().mockResolvedValue({ success: false, error: 'not used', usage: {} }), + runSynthesis: vi.fn().mockImplementation(async (req: SynthesisRunRequest) => { + capturedRequests.push(req); + // Return a valid outline with the correct sourceHash for the collected source. + const outline = { ...fakeOutline, sourceHash: req.prompt.match(/sourceHash "([^"]+)"/)?.[1] ?? fakeOutline.sourceHash }; + return { success: true as const, data: outline, usage: { inputTokens: 10, outputTokens: 5, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, cacheCreation5mInputTokens: 0, cacheCreation1hInputTokens: 0, webSearchRequests: 0, costUSD: 0 } }; + }), + }; + + await buildSkillOutline({ + skill: { name: 'security', description: 'Security skill', prompt: 'Find security issues.', rootDir }, + runtime: mockRuntime, + providers, + regenerate: true, + }); + + expect(capturedRequests).toHaveLength(1); + const req = capturedRequests[0]!; + // providerOptions must be forwarded: it is a PiProviderOptions object with providers[0].name === 'litellm' + expect(req.providerOptions).toBeDefined(); + const po = req.providerOptions as { providers: { name: string }[] }; + expect(po.providers[0]!.name).toBe('litellm'); + }); +}); diff --git a/packages/warden/src/skill-builder/outline.ts b/packages/warden/src/skill-builder/outline.ts index b207f6cb..6fa5975a 100644 --- a/packages/warden/src/skill-builder/outline.ts +++ b/packages/warden/src/skill-builder/outline.ts @@ -1,9 +1,9 @@ import { createHash } from 'node:crypto'; import { existsSync } from 'node:fs'; import { basename } from 'node:path'; -import type { SkillDefinition } from '../config/schema.js'; +import type { ProvidersConfig, SkillDefinition } from '../config/schema.js'; import type { Runtime, RuntimeName } from '../sdk/runtimes/index.js'; -import { getRuntime } from '../sdk/runtimes/index.js'; +import { getRuntime, getRuntimeProviderOptions } from '../sdk/runtimes/index.js'; import { runStructuredSkillBuilderAgent, StructuredSkillBuilderAgentError } from './agentic.js'; import { GENERATED_SKILL_DEFINITION_FILE, @@ -51,6 +51,7 @@ export interface BuildSkillOutlineOptions { repairMaxRetries?: number; onStatus?: (message: string) => void; source?: SkillBuildSource; + providers?: ProvidersConfig; } export class SkillBuildOutlineError extends Error { @@ -359,6 +360,7 @@ export async function buildSkillOutline( maxTurns: options.maxTurns ?? SKILL_BUILD_MAX_TURNS, apiKey, abortController: options.abortController, + providers: options.providers, repair: { apiKey, model: options.repairModel, @@ -408,6 +410,7 @@ export async function buildSkillOutline( maxTokens: SKILL_BUILD_MAX_TOKENS, timeout: SKILL_BUILD_TIMEOUT_MS, maxRetries, + providerOptions: getRuntimeProviderOptions(runtime.name, { providers: options.providers }), }); if (!result.success) { diff --git a/packages/warden/src/skill-builder/skill.ts b/packages/warden/src/skill-builder/skill.ts index 38f47147..9e8e75ab 100644 --- a/packages/warden/src/skill-builder/skill.ts +++ b/packages/warden/src/skill-builder/skill.ts @@ -3,6 +3,7 @@ import { existsSync } from 'node:fs'; import { performance } from 'node:perf_hooks'; import { parse as parseYaml } from 'yaml'; import { aggregateUsage } from '../sdk/usage.js'; +import type { ProvidersConfig } from '../config/schema.js'; import type { Runtime } from '../sdk/runtimes/index.js'; import { runStructuredSkillBuilderAgent, StructuredSkillBuilderAgentError } from './agentic.js'; import type { UsageStats } from '../types/index.js'; @@ -462,6 +463,7 @@ export async function buildGeneratedSkill(args: { repairMaxRetries?: number; authoringSkillRoot?: string; onStatus?: (message: string) => void; + providers?: ProvidersConfig; }): Promise { const startedAt = performance.now(); const statePath = getBuildStatePath(args.rootDir); @@ -522,6 +524,7 @@ export async function buildGeneratedSkill(args: { maxTurns, apiKey: args.apiKey, abortController: args.abortController, + providers: args.providers, repair, }); @@ -551,6 +554,7 @@ export async function buildGeneratedSkill(args: { apiKey: args.apiKey, writeAccess: true, abortController: args.abortController, + providers: args.providers, repair, }); @@ -598,6 +602,7 @@ export async function buildGeneratedSkill(args: { maxTurns: Math.min(maxTurns, defaultValidationMaxTurns()), apiKey: args.apiKey, abortController: args.abortController, + providers: args.providers, repair, }); reviewResults.push(review); @@ -637,6 +642,7 @@ export async function buildGeneratedSkill(args: { apiKey: args.apiKey, writeAccess: true, abortController: args.abortController, + providers: args.providers, repair, }); revisionResults.push(revision); From a1b5583cdd5cf9cc4143171ba34b91c65e3e6775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Sat, 27 Jun 2026 00:54:36 +0200 Subject: [PATCH 12/22] feat(runtime): fail fast when a remote custom provider has no key Co-Authored-By: Claude Opus 4.8 --- .../warden/src/action/triggers/executor.ts | 8 +++ packages/warden/src/cli/main.test.ts | 49 +++++++++++++++++++ packages/warden/src/cli/main.ts | 32 ++++++++++++ 3 files changed, 89 insertions(+) diff --git a/packages/warden/src/action/triggers/executor.ts b/packages/warden/src/action/triggers/executor.ts index 2b4360bb..846a1818 100644 --- a/packages/warden/src/action/triggers/executor.ts +++ b/packages/warden/src/action/triggers/executor.ts @@ -24,6 +24,10 @@ import type { Semaphore } from '../../utils/index.js'; import { Verbosity } from '../../cli/output/verbosity.js'; import type { ProviderFailureCircuitBreaker } from '../../sdk/circuit-breaker.js'; import { assertValidPiModelSelectors } from '../../sdk/runtimes/model-selectors.js'; +import { + buildPiProviderOptions, + assertCustomProviderAuth, +} from '../../sdk/runtimes/custom-provider.js'; import { captureActionTriggerError } from '../error-reporting.js'; /** Log-mode output for CI: no TTY, no color. */ @@ -174,6 +178,10 @@ export async function executeTrigger( try { assertValidPiModelSelectors([trigger]); + if ((trigger.runtime ?? 'pi') === 'pi') { + assertCustomProviderAuth(buildPiProviderOptions(trigger.providers, process.env)); + } + const taskOptions: SkillTaskOptions = { name: trigger.name, displayName: trigger.skill, diff --git a/packages/warden/src/cli/main.test.ts b/packages/warden/src/cli/main.test.ts index 0b7322c7..3f96fbab 100644 --- a/packages/warden/src/cli/main.test.ts +++ b/packages/warden/src/cli/main.test.ts @@ -18,6 +18,7 @@ import { appendReportToRunLog, buildFinalChunkRecords, renderFinalRunLogContent, + verifyCustomProviderAuthForRun, type RunLog, type RunSkillSpec, } from './main.js'; @@ -828,3 +829,51 @@ describe('resolveInvocationCwd', () => { expect(resolveInvocationCwd('/launcher', '../repo')).toBe('/repo'); }); }); + +describe('verifyCustomProviderAuthForRun', () => { + const remoteProviders = { + litellm: { + baseUrl: 'https://gw.example.com/v1', + api: 'openai-completions' as const, + models: [{ id: 'my-model' }], + }, + }; + + const loopbackProviders = { + local: { + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions' as const, + models: [{ id: 'my-model' }], + }, + }; + + function fakeReporter() { + return { error: vi.fn() } as unknown as InstanceType; + } + + it('returns false and calls reporter.error when a pi item has a remote provider with no key', () => { + const reporter = fakeReporter(); + const items = [{ runtime: 'pi' as const, providers: remoteProviders }]; + // No key in env — process.env should not have WARDEN_LITELLM_API_KEY or LITELLM_API_KEY set + const result = verifyCustomProviderAuthForRun(items, reporter); + expect(result).toBe(false); + expect(reporter.error).toHaveBeenCalledOnce(); + expect((reporter.error as ReturnType).mock.calls[0]![0]).toContain('litellm'); + }); + + it('returns true when a pi item has a loopback provider with no key', () => { + const reporter = fakeReporter(); + const items = [{ runtime: 'pi' as const, providers: loopbackProviders }]; + const result = verifyCustomProviderAuthForRun(items, reporter); + expect(result).toBe(true); + expect(reporter.error).not.toHaveBeenCalled(); + }); + + it('returns true and skips auth check for claude runtime items even with remote providers', () => { + const reporter = fakeReporter(); + const items = [{ runtime: 'claude' as const, providers: remoteProviders }]; + const result = verifyCustomProviderAuthForRun(items, reporter); + expect(result).toBe(true); + expect(reporter.error).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/warden/src/cli/main.ts b/packages/warden/src/cli/main.ts index ae3e7ef7..2b805481 100644 --- a/packages/warden/src/cli/main.ts +++ b/packages/warden/src/cli/main.ts @@ -10,6 +10,10 @@ import { invalidPiModelSelectorMessage, type InvalidPiModelSelector, } from '../sdk/runtimes/model-selectors.js'; +import { + buildPiProviderOptions, + assertCustomProviderAuth, +} from '../sdk/runtimes/custom-provider.js'; import { mapExtractionErrorCode } from '../sdk/errors.js'; import { aggregateAuxiliaryUsageAttribution, mergeAuxiliaryUsage } from '../sdk/usage.js'; import { resolveSkillAsync, SkillLoaderError } from '../skills/loader.js'; @@ -804,6 +808,26 @@ function verifyClaudeAuthForRun(args: { } } +/** + * Verify that every pi-runtime item with a remote custom provider has a resolvable + * API key before analysis starts. Returns false (and emits an error) on first failure. + */ +export function verifyCustomProviderAuthForRun( + items: { runtime?: SkillRunnerOptions['runtime']; providers?: SkillRunnerOptions['providers'] }[], + reporter: Reporter, +): boolean { + for (const item of items) { + if ((item.runtime ?? 'pi') !== 'pi') continue; + try { + assertCustomProviderAuth(buildPiProviderOptions(item.providers, process.env)); + } catch (error) { + reporter.error(error instanceof Error ? error.message : String(error)); + return false; + } + } + return true; +} + function renderSkillRunHeader(args: { reporter: Reporter; skill: SkillDefinition; @@ -1164,6 +1188,10 @@ export async function runSkills( return 1; } + if (!verifyCustomProviderAuthForRun(skillsToRun, reporter)) { + return 1; + } + // Build skill tasks // Model precedence: defaults.agent.model > defaults.model > CLI flag > WARDEN_MODEL env var > SDK default // sdkModel is undefined when no model is explicitly configured (lets SDK use its default). @@ -1505,6 +1533,10 @@ async function runConfigMode(options: CLIOptions, reporter: Reporter): Promise ({ From 0843669fb012672fb3568246a4d59d14a190aeee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Sat, 27 Jun 2026 00:58:27 +0200 Subject: [PATCH 13/22] test(cli): make custom-provider preflight tests hermetic Inject env into verifyCustomProviderAuthForRun so the preflight tests no longer depend on the runner's environment (a CI-set LITELLM_API_KEY would otherwise false-pass the no-key case). Co-Authored-By: Claude Opus 4.8 --- packages/warden/src/cli/main.test.ts | 8 ++++---- packages/warden/src/cli/main.ts | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/warden/src/cli/main.test.ts b/packages/warden/src/cli/main.test.ts index 3f96fbab..1a2c45ef 100644 --- a/packages/warden/src/cli/main.test.ts +++ b/packages/warden/src/cli/main.test.ts @@ -854,8 +854,8 @@ describe('verifyCustomProviderAuthForRun', () => { it('returns false and calls reporter.error when a pi item has a remote provider with no key', () => { const reporter = fakeReporter(); const items = [{ runtime: 'pi' as const, providers: remoteProviders }]; - // No key in env — process.env should not have WARDEN_LITELLM_API_KEY or LITELLM_API_KEY set - const result = verifyCustomProviderAuthForRun(items, reporter); + // Explicit empty env keeps the test hermetic regardless of the runner's environment. + const result = verifyCustomProviderAuthForRun(items, reporter, {}); expect(result).toBe(false); expect(reporter.error).toHaveBeenCalledOnce(); expect((reporter.error as ReturnType).mock.calls[0]![0]).toContain('litellm'); @@ -864,7 +864,7 @@ describe('verifyCustomProviderAuthForRun', () => { it('returns true when a pi item has a loopback provider with no key', () => { const reporter = fakeReporter(); const items = [{ runtime: 'pi' as const, providers: loopbackProviders }]; - const result = verifyCustomProviderAuthForRun(items, reporter); + const result = verifyCustomProviderAuthForRun(items, reporter, {}); expect(result).toBe(true); expect(reporter.error).not.toHaveBeenCalled(); }); @@ -872,7 +872,7 @@ describe('verifyCustomProviderAuthForRun', () => { it('returns true and skips auth check for claude runtime items even with remote providers', () => { const reporter = fakeReporter(); const items = [{ runtime: 'claude' as const, providers: remoteProviders }]; - const result = verifyCustomProviderAuthForRun(items, reporter); + const result = verifyCustomProviderAuthForRun(items, reporter, {}); expect(result).toBe(true); expect(reporter.error).not.toHaveBeenCalled(); }); diff --git a/packages/warden/src/cli/main.ts b/packages/warden/src/cli/main.ts index 2b805481..bd9025b9 100644 --- a/packages/warden/src/cli/main.ts +++ b/packages/warden/src/cli/main.ts @@ -815,11 +815,12 @@ function verifyClaudeAuthForRun(args: { export function verifyCustomProviderAuthForRun( items: { runtime?: SkillRunnerOptions['runtime']; providers?: SkillRunnerOptions['providers'] }[], reporter: Reporter, + env: NodeJS.ProcessEnv = process.env, ): boolean { for (const item of items) { if ((item.runtime ?? 'pi') !== 'pi') continue; try { - assertCustomProviderAuth(buildPiProviderOptions(item.providers, process.env)); + assertCustomProviderAuth(buildPiProviderOptions(item.providers, env)); } catch (error) { reporter.error(error instanceof Error ? error.message : String(error)); return false; From 4a7dbc8100ec586dee8d72cf4f4595debfff1821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Sat, 27 Jun 2026 00:59:46 +0200 Subject: [PATCH 14/22] docs: document self-hosted OpenAI-compatible providers (LiteLLM) Co-Authored-By: Claude Opus 4.8 --- .../docs/src/content/docs/config/models.mdx | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/packages/docs/src/content/docs/config/models.mdx b/packages/docs/src/content/docs/config/models.mdx index 4a79f08e..4cf17999 100644 --- a/packages/docs/src/content/docs/config/models.mdx +++ b/packages/docs/src/content/docs/config/models.mdx @@ -56,6 +56,52 @@ Warden mirrors these to the native `{PROVIDER}_API_KEY` expected by each SDK at If you already have a native provider key set (e.g. `OPENAI_API_KEY`), Warden will use it directly and the `WARDEN_`-prefixed form is not required. +## Self-hosted / OpenAI-compatible providers (LiteLLM) + +When `runtime = "pi"`, you can register a self-hosted, OpenAI-compatible endpoint +(such as a [LiteLLM](https://docs.litellm.ai/) proxy) as a named provider and +target its models with the usual `provider/model` selector. The custom provider +covers every model lane (agent, auxiliary, synthesis). + +```toml title="warden.toml" +[defaults] +runtime = "pi" + +[defaults.providers.litellm] +baseUrl = "http://localhost:4000/v1" # required; OpenAI-compatible base URL +api = "openai-completions" # optional; default +# headers = { "X-Tenant" = "team-a" } # optional custom headers +# apiKeyEnv = "WARDEN_LITELLM_API_KEY" # optional; overrides the default lookup + +[[defaults.providers.litellm.models]] +id = "my-model" # required; the model name your endpoint exposes +# contextWindow = 128000 # optional; default 128000 +# maxTokens = 8192 # optional; default 8192 +# reasoning = false # optional; default false +# cost = { input = 0, output = 0, cacheRead = 0, cacheWrite = 0 } + +[defaults.agent] +model = "litellm/my-model" +``` + +**Model defaults:** only `id` is required. Warden fills `name` (= `id`), +`reasoning` (`false`), `input` (`["text"]`), `contextWindow` (`128000`), +`maxTokens` (`8192`), and `cost` (all zeros). Self-hosted runs therefore report +`$0` cost unless you set explicit costs. + +**Authentication:** the API key is read from the environment, never from +`warden.toml`. Warden looks up `apiKeyEnv` if set, otherwise +`WARDEN__API_KEY` then `_API_KEY` (e.g. `WARDEN_LITELLM_API_KEY`). +A `localhost` / `127.0.0.1` base URL may run without a key; any other host +requires one or Warden fails before analysis starts. + +### Alternative: Pi `models.json` + +Advanced users can instead define a custom provider in Pi's own `models.json` +(the format Pi loads on startup). Warden's Pi runtime picks up any providers and +models defined there. Prefer the `warden.toml` block above unless you already +maintain a shared Pi configuration. + ## Claude Runtime Models When `runtime = "claude"`, use the model IDs accepted by Claude Code: From 9e36ca81c1946443f98c7982ae688dbb10e8f7cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Sat, 27 Jun 2026 01:06:12 +0200 Subject: [PATCH 15/22] docs,runtime: clarify env-resolution boundary and IPv6 loopback Address final-review nits: document the intentional process.env read in getRuntimeProviderOptions (preflight resolves against the same env), and include ::1 in the loopback auth note. Co-Authored-By: Claude Opus 4.8 --- packages/docs/src/content/docs/config/models.mdx | 4 ++-- packages/warden/src/sdk/runtimes/index.ts | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/docs/src/content/docs/config/models.mdx b/packages/docs/src/content/docs/config/models.mdx index 4cf17999..c8ba309f 100644 --- a/packages/docs/src/content/docs/config/models.mdx +++ b/packages/docs/src/content/docs/config/models.mdx @@ -92,8 +92,8 @@ model = "litellm/my-model" **Authentication:** the API key is read from the environment, never from `warden.toml`. Warden looks up `apiKeyEnv` if set, otherwise `WARDEN__API_KEY` then `_API_KEY` (e.g. `WARDEN_LITELLM_API_KEY`). -A `localhost` / `127.0.0.1` base URL may run without a key; any other host -requires one or Warden fails before analysis starts. +A loopback base URL (`localhost`, `127.0.0.1`, or `::1`) may run without a key; +any other host requires one or Warden fails before analysis starts. ### Alternative: Pi `models.json` diff --git a/packages/warden/src/sdk/runtimes/index.ts b/packages/warden/src/sdk/runtimes/index.ts index 90d46cfe..94a1cee7 100644 --- a/packages/warden/src/sdk/runtimes/index.ts +++ b/packages/warden/src/sdk/runtimes/index.ts @@ -53,6 +53,9 @@ export function getRuntimeProviderOptions( } if (name === 'pi') { + // Resolve provider API keys from the live process env at the runtime + // boundary. Preflight (verifyCustomProviderAuthForRun) resolves against the + // same env, so both phases agree as long as env is not mutated mid-run. return buildPiProviderOptions(options.providers, process.env); } From 033c09a2e8a289041f3923e3278b0f52ff944a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Tue, 30 Jun 2026 10:45:43 +0200 Subject: [PATCH 16/22] fix(runtime): apply custom-provider preflight consistently across entry points Address PR #1 review findings for the LiteLLM custom-provider feature: - schedule.ts: scheduled workflows ran assertValidPiModelSelectors but skipped the assertCustomProviderAuth preflight that the trigger executor and CLI already perform. Add it for pi-runtime triggers so a missing remote provider key fails fast instead of at run time. - main.ts (runConfigMode): the custom-provider preflight used raw trigger.runtime, ignoring the --runtime override that the adjacent Claude auth check and the actual run already honor. Thread `options.runtime ?? trigger.runtime` so preflight matches what executes. - judge.runtime-options.test.ts: add coverage proving providerOptions flow from evaluateFix through to runAuxiliary (previously unverified). - outline.test.ts: add a repoPath case exercising the runStructuredSkillBuilderAgent branch; provider forwarding there was untested (only the runSynthesis branch was covered). Co-Authored-By: Claude Opus 4.8 --- .../judge.runtime-options.test.ts | 62 ++++++++++++++ .../warden/src/action/workflow/schedule.ts | 8 ++ packages/warden/src/cli/main.ts | 6 +- .../warden/src/skill-builder/outline.test.ts | 81 +++++++++++++++++++ 4 files changed, 156 insertions(+), 1 deletion(-) diff --git a/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts b/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts index a2920217..4ab2d07f 100644 --- a/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts +++ b/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { Octokit } from '@octokit/rest'; import type { ExistingComment } from '../../output/dedup.js'; +import type { ProvidersConfig } from '../../config/schema.js'; import type { FixJudgeContext, FixJudgeInput, FixJudgeRuntimeOptions } from './judge.js'; describe('evaluateFix runtime options', () => { @@ -69,7 +70,68 @@ describe('evaluateFix runtime options', () => { expect(runAuxiliary).toHaveBeenCalledWith( expect.objectContaining({ prompt: expect.stringContaining(''), + providerOptions: undefined, }) ); }); + + it('forwards resolved provider options to runAuxiliary', async () => { + const runAuxiliary = vi.fn().mockResolvedValue({ + success: true, + data: { status: 'not_attempted', reasoning: 'No related changes' }, + usage: { inputTokens: 0, outputTokens: 0, costUSD: 0 }, + }); + const getRuntime = vi.fn(() => ({ runAuxiliary })); + const providerOptions = { providers: [{ name: 'litellm' }] }; + const getRuntimeProviderOptions = vi.fn(() => providerOptions); + + vi.doMock('../../sdk/runtimes/index.js', () => ({ + getRuntime, + getRuntimeProviderOptions, + })); + + const { evaluateFix } = await import('./judge.js'); + + const comment: ExistingComment = { + id: 1, + path: 'src/handler.ts', + line: 12, + title: 'SQL injection', + description: 'User input is concatenated into SQL', + contentHash: 'abc123', + isWarden: true, + threadId: 'thread-1', + }; + + const input: FixJudgeInput = { + comment, + changedFiles: ['src/handler.ts'], + codeBeforeFix: '12: const query = "SELECT * FROM users WHERE id = " + id;', + }; + + const context: FixJudgeContext = { + octokit: {} as Octokit, + owner: 'test-owner', + repo: 'test-repo', + baseSha: 'base123', + headSha: 'head456', + patches: new Map(), + }; + + const providers: ProvidersConfig = { + litellm: { + baseUrl: 'http://localhost:4000', + api: 'openai-completions', + models: [{ id: 'litellm/gpt-4o', name: 'gpt-4o' }], + }, + }; + + await evaluateFix(input, context, 'api-key', { runtime: 'pi', providers }); + + expect(getRuntime).toHaveBeenCalledWith('pi'); + expect(getRuntimeProviderOptions).toHaveBeenCalledWith('pi', { providers }); + expect(runAuxiliary).toHaveBeenCalledWith( + expect.objectContaining({ providerOptions }) + ); + }); }); diff --git a/packages/warden/src/action/workflow/schedule.ts b/packages/warden/src/action/workflow/schedule.ts index c4488884..eb251e9b 100644 --- a/packages/warden/src/action/workflow/schedule.ts +++ b/packages/warden/src/action/workflow/schedule.ts @@ -16,6 +16,10 @@ import type { ScheduleConfig } from '../../config/schema.js'; import { buildScheduleEventContext } from '../../event/schedule-context.js'; import { runSkill } from '../../sdk/runner.js'; import { assertValidPiModelSelectors } from '../../sdk/runtimes/model-selectors.js'; +import { + buildPiProviderOptions, + assertCustomProviderAuth, +} from '../../sdk/runtimes/custom-provider.js'; import { createOrUpdateIssue } from '../../output/github-issues.js'; import { shouldFail, countFindingsAtOrAbove, countSeverity } from '../../triggers/matcher.js'; import { resolveSkillAsync } from '../../skills/loader.js'; @@ -179,6 +183,10 @@ async function runScheduleWorkflowInner( try { assertValidPiModelSelectors([resolved]); + if ((resolved.runtime ?? 'pi') === 'pi') { + assertCustomProviderAuth(buildPiProviderOptions(resolved.providers, process.env)); + } + // Build context from paths filter const patterns = resolved.filters?.paths ?? ['**/*']; const ignorePatterns = resolved.filters?.ignorePaths; diff --git a/packages/warden/src/cli/main.ts b/packages/warden/src/cli/main.ts index bd9025b9..b63296ae 100644 --- a/packages/warden/src/cli/main.ts +++ b/packages/warden/src/cli/main.ts @@ -1534,7 +1534,11 @@ async function runConfigMode(options: CLIOptions, reporter: Reporter): Promise ({ + runtime: options.runtime ?? trigger.runtime, + providers: trigger.providers, + })); + if (!verifyCustomProviderAuthForRun(customProviderItems, reporter)) { return 1; } diff --git a/packages/warden/src/skill-builder/outline.test.ts b/packages/warden/src/skill-builder/outline.test.ts index 121584a9..96c8125b 100644 --- a/packages/warden/src/skill-builder/outline.test.ts +++ b/packages/warden/src/skill-builder/outline.test.ts @@ -250,4 +250,85 @@ describe('buildSkillOutline: custom providers forwarding', () => { const po = req.providerOptions as { providers: { name: string }[] }; expect(po.providers[0]!.name).toBe('litellm'); }); + + it('forwards providerOptions through the repoPath agentic branch', async () => { + const rootDir = mkdtempSync(join(tmpdir(), 'warden-outline-providers-repo-')); + tempDirs.push(rootDir); + writeFileSync(join(rootDir, 'warden.yaml'), `version: 1\nkind: generated-skill\nname: security\nprompt: Find security issues.\n`, 'utf-8'); + + const providers: ProvidersConfig = { + litellm: { + baseUrl: 'http://localhost:4000', + api: 'openai-completions', + models: [{ id: 'litellm/gpt-4o', name: 'gpt-4o' }], + }, + }; + + const fakeOutline: SkillBuildOutline = { + version: SKILL_BUILD_OUTLINE_SCHEMA_VERSION, + skill: 'security', + sourceHash: 'placeholder', + buildVersion: SKILL_BUILD_VERSION, + scopeProfile: { + kind: 'domain', + subject: 'Generic security review', + localContextUsed: false, + observedContext: ['Generic security review'], + unresolvedContext: [], + }, + build: { + phases: [{ id: 'build-tracks', status: 'generated' }], + }, + tracks: [{ + id: 'auth-bypass', + title: 'Authentication bypasses', + goal: 'Find broken authentication checks.', + rationale: 'Authentication bugs are core security issues.', + sourceSignals: ['Auth endpoints'], + owns: ['Missing auth checks'], + excludes: [], + relevanceSignals: ['Session checks'], + evidenceFocus: ['Changed auth conditions'], + checks: ['Trace auth preconditions'], + safeCounterpatterns: ['Explicit user verification'], + falsePositiveTraps: ['Defense-in-depth logging'], + researchHints: [], + }], + }; + + // The agentic branch (repoPath set) runs through runtime.runSkill and parses + // the structured outline from result.text. Echo the correct sourceHash back so + // validateOutlineIdentity passes. + const capturedRequests: { providerOptions?: unknown }[] = []; + const mockRuntime: Runtime = { + name: 'pi', + runSkill: vi.fn().mockImplementation(async (req: { userPrompt: string; providerOptions?: unknown }) => { + capturedRequests.push({ providerOptions: req.providerOptions }); + const sourceHash = req.userPrompt.match(/sourceHash "([^"]+)"/)?.[1] ?? fakeOutline.sourceHash; + return { + result: { + status: 'success' as const, + text: JSON.stringify({ ...fakeOutline, sourceHash }), + errors: [], + usage: { inputTokens: 10, outputTokens: 5, costUSD: 0 }, + }, + }; + }), + runAuxiliary: vi.fn().mockResolvedValue({ success: false, error: 'not used', usage: {} }), + runSynthesis: vi.fn().mockResolvedValue({ success: false, error: 'should not be called', usage: {} }), + }; + + await buildSkillOutline({ + skill: { name: 'security', description: 'Security skill', prompt: 'Find security issues.', rootDir }, + runtime: mockRuntime, + repoPath: rootDir, + providers, + regenerate: true, + }); + + expect(mockRuntime.runSynthesis).not.toHaveBeenCalled(); + expect(capturedRequests).toHaveLength(1); + const po = capturedRequests[0]!.providerOptions as { providers: { name: string }[] }; + expect(po.providers[0]!.name).toBe('litellm'); + }); }); From f4af42af7cf572f3e84fef8e0909349b90b1e988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Tue, 30 Jun 2026 11:52:05 +0200 Subject: [PATCH 17/22] fix(runtime): inherit the default model for auxiliary and synthesis lanes The auxiliary lane (structured extraction, dedup, merge, fix evaluation) and the synthesis lane resolved their model only from [defaults.auxiliary] / [defaults.synthesis]. When those were unset, the lanes fell back to the runtime's built-in default model, which for the Pi runtime resolves against a different provider entirely (e.g. gemini via Google's API). For self-hosted custom providers this is a real defect, not just a docs gap: configuring `defaults.model = "litellm/..."` kept only the agent lane on the proxy while extraction/dedup/synthesis silently escaped to an unconfigured external provider. That both breaks (auth/quota errors mid-run) and, worse, leaks model output carrying code context off the self-hosted endpoint the user deliberately chose. Make the auxiliary and synthesis lanes fall back to the resolved global default model (defaults.agent.model -> defaults.model -> --model -> WARDEN_MODEL) when their own model is unset, in both resolution sites (CLI resolveCliDefault* and loader resolveSkillConfigs). Explicit auxiliary/synthesis models still win, and lanes still ignore skill/trigger-level overrides (agent lane only). A single configured model now drives every lane. Co-Authored-By: Claude Opus 4.8 --- .../docs/src/content/docs/config/models.mdx | 15 +++++++-- packages/warden/src/cli/main.test.ts | 32 +++++++++++++++++++ packages/warden/src/cli/main.ts | 24 ++++++++++---- packages/warden/src/config/loader.test.ts | 29 +++++++++++++++++ packages/warden/src/config/loader.ts | 13 +++++++- 5 files changed, 103 insertions(+), 10 deletions(-) diff --git a/packages/docs/src/content/docs/config/models.mdx b/packages/docs/src/content/docs/config/models.mdx index c8ba309f..c84a0f40 100644 --- a/packages/docs/src/content/docs/config/models.mdx +++ b/packages/docs/src/content/docs/config/models.mdx @@ -202,8 +202,19 @@ Main agent model precedence, from highest to lowest: | `WARDEN_MODEL` | Environment fallback. | | SDK/runtime default | Used when no explicit model is set. | -Auxiliary and synthesis models only come from `[defaults.auxiliary]` and -`[defaults.synthesis]`. They do not inherit skill or trigger `model` overrides. +The auxiliary lane (structured extraction, dedup, merge, fix evaluation) and the +synthesis lane prefer `[defaults.auxiliary]` and `[defaults.synthesis]`. When +unset, they fall back to the global default model +(`defaults.agent.model` → `defaults.model` → `--model` → `WARDEN_MODEL`), so a +single configured model drives every lane. Synthesis falls back to the auxiliary +model before the global default. These lanes do not inherit skill- or +trigger-level `model` overrides, which apply to the agent lane only. + +This matters for self-hosted providers: setting just `defaults.model` to a +custom-provider model (e.g. `litellm/...`) keeps the auxiliary and synthesis +lanes on that provider too, rather than escaping to a runtime default on another +provider. Set `[defaults.auxiliary]` explicitly only when you want those lanes on +a different (e.g. cheaper) model. Effort is separate from model selection. For local runs, `--effort` overrides `defaults.agent.effort` for that diff --git a/packages/warden/src/cli/main.test.ts b/packages/warden/src/cli/main.test.ts index 1a2c45ef..ed923a7d 100644 --- a/packages/warden/src/cli/main.test.ts +++ b/packages/warden/src/cli/main.test.ts @@ -667,6 +667,38 @@ describe('resolveCliDefaultModel', () => { expect(model).toBeUndefined(); }); + it('inherits the top-level model for the auxiliary lane when unset', () => { + expect( + resolveCliDefaultAuxiliaryModel({ defaults: { model: 'litellm/gemma' } }), + ).toBe('litellm/gemma'); + }); + + it('inherits the agent model for the auxiliary lane when unset', () => { + expect( + resolveCliDefaultAuxiliaryModel({ defaults: { agent: { model: 'litellm/gemma' } } }), + ).toBe('litellm/gemma'); + }); + + it('inherits the --model flag for the auxiliary lane when nothing else is set', () => { + expect( + resolveCliDefaultAuxiliaryModel({ defaults: {} }, 'litellm/gemma'), + ).toBe('litellm/gemma'); + }); + + it('prefers an explicit auxiliary model over the inherited top-level model', () => { + expect( + resolveCliDefaultAuxiliaryModel({ + defaults: { model: 'litellm/gemma', auxiliary: { model: 'litellm/cheap' } }, + }), + ).toBe('litellm/cheap'); + }); + + it('inherits the top-level model for synthesis through the auxiliary fallback', () => { + expect( + resolveCliDefaultSynthesisModel({ defaults: { model: 'litellm/gemma' } }), + ).toBe('litellm/gemma'); + }); + it('prefers synthesis defaults and falls back to auxiliary defaults', () => { const explicit = resolveCliDefaultSynthesisModel({ defaults: { diff --git a/packages/warden/src/cli/main.ts b/packages/warden/src/cli/main.ts index b63296ae..9ade494d 100644 --- a/packages/warden/src/cli/main.ts +++ b/packages/warden/src/cli/main.ts @@ -929,20 +929,30 @@ export function resolveCliDefaultModel( ); } -/** Resolve the default auxiliary model used for helper and repair passes. */ +/** + * Resolve the default auxiliary model used for helper and repair passes. + * Falls back to the resolved agent/top-level model so a single configured + * model drives every lane (and self-hosted providers stay self-contained + * instead of escaping to a runtime default on another provider). + */ export function resolveCliDefaultAuxiliaryModel( - config: Pick | null | undefined + config: Pick | null | undefined, + cliModel?: string ): string | undefined { - return emptyToUndefined(config?.defaults?.auxiliary?.model); + return ( + emptyToUndefined(config?.defaults?.auxiliary?.model) ?? + resolveCliDefaultModel(config, cliModel) + ); } /** Resolve the default synthesis model, falling back to the auxiliary lane when unset. */ export function resolveCliDefaultSynthesisModel( - config: Pick | null | undefined + config: Pick | null | undefined, + cliModel?: string ): string | undefined { return ( emptyToUndefined(config?.defaults?.synthesis?.model) ?? - resolveCliDefaultAuxiliaryModel(config) + resolveCliDefaultAuxiliaryModel(config, cliModel) ); } @@ -1100,8 +1110,8 @@ export async function runSkills( const repoPath = findRepoPath(cwd); const config = loadOptionalConfig(options, repoPath); const defaultModel = resolveCliDefaultModel(config, options.model); - const defaultAuxiliaryModel = resolveCliDefaultAuxiliaryModel(config); - const defaultSynthesisModel = resolveCliDefaultSynthesisModel(config); + const defaultAuxiliaryModel = resolveCliDefaultAuxiliaryModel(config, options.model); + const defaultSynthesisModel = resolveCliDefaultSynthesisModel(config, options.model); const defaultEffort = resolveCliEffort(config, options.effort); const defaultRuntime = options.runtime ?? config?.defaults?.runtime ?? 'pi'; const pathToClaudeCodeExecutable = resolveClaudeCodeExecutablePath(); diff --git a/packages/warden/src/config/loader.test.ts b/packages/warden/src/config/loader.test.ts index 0f05bf66..ae057115 100644 --- a/packages/warden/src/config/loader.test.ts +++ b/packages/warden/src/config/loader.test.ts @@ -509,6 +509,35 @@ describe('resolveSkillConfigs', () => { expect(resolved?.auxiliaryModel).toBe('claude-haiku-4-5'); expect(resolved?.synthesisModel).toBe('claude-haiku-4-5'); }); + + it('inherits the top-level model for the auxiliary and synthesis lanes when unset', () => { + const config: WardenConfig = { + ...baseConfig, + defaults: { + model: 'litellm/gemma-4-12b-coder', + }, + }; + + const [resolved] = resolveSkillConfigs(config); + + expect(resolved?.auxiliaryModel).toBe('litellm/gemma-4-12b-coder'); + expect(resolved?.synthesisModel).toBe('litellm/gemma-4-12b-coder'); + }); + + it('prefers explicit auxiliary and synthesis models over the inherited top-level model', () => { + const config: WardenConfig = { + ...baseConfig, + defaults: { + model: 'litellm/gemma-4-12b-coder', + auxiliary: { model: 'litellm/cheap' }, + }, + }; + + const [resolved] = resolveSkillConfigs(config); + + expect(resolved?.auxiliaryModel).toBe('litellm/cheap'); + expect(resolved?.synthesisModel).toBe('litellm/cheap'); + }); }); describe('minConfidence merge', () => { diff --git a/packages/warden/src/config/loader.ts b/packages/warden/src/config/loader.ts index bc36140a..7b73d434 100644 --- a/packages/warden/src/config/loader.ts +++ b/packages/warden/src/config/loader.ts @@ -497,7 +497,18 @@ export function resolveSkillConfigs( const result: ResolvedTrigger[] = []; const runtime = defaults?.runtime ?? 'pi'; const providers = defaults?.providers; - const auxiliaryModel = emptyToUndefined(defaults?.auxiliary?.model); + // Default agent/top-level model, used as the fallback for the auxiliary and + // synthesis lanes so one configured model drives every lane (and self-hosted + // providers stay self-contained instead of escaping to a runtime default on + // another provider). Explicit auxiliary/synthesis models still win. + const defaultAgentModel = + emptyToUndefined(defaults?.agent?.model) ?? + emptyToUndefined(defaults?.model) ?? + emptyToUndefined(cliModel) ?? + envModel; + const auxiliaryModel = + emptyToUndefined(defaults?.auxiliary?.model) ?? + defaultAgentModel; const synthesisModel = emptyToUndefined(defaults?.synthesis?.model) ?? auxiliaryModel; From c91eff67d0e276ea35feea5adcab4804fb9e937a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Tue, 30 Jun 2026 16:34:38 +0200 Subject: [PATCH 18/22] fix(action): inherit the default model for the PR workflow auxiliary lane resolveWorkflowAuxiliaryOptions read only [defaults.auxiliary].model and never fell back to the global default model, unlike resolveSkillConfigs and the CLI resolvers. In the GitHub Action PR workflow, dedup, consolidation, and fix evaluation could still hit the runtime's built-in default model on another provider while `providers` pointed at a custom (e.g. self-hosted) endpoint - the same escape the lane-inheritance fix closed for the other entry points. Add the global-model fallback (agent.model -> model) after the explicit auxiliary models, keeping the existing base-first enforced-baseline precedence. Export the resolver and unit-test the inheritance and precedence. Reported by Cursor Bugbot on PR #416. Co-Authored-By: Claude Opus 4.8 --- .../src/action/workflow/pr-workflow.test.ts | 45 ++++++++++++++++++- .../warden/src/action/workflow/pr-workflow.ts | 12 ++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/packages/warden/src/action/workflow/pr-workflow.test.ts b/packages/warden/src/action/workflow/pr-workflow.test.ts index bb7d2cf0..dc8f7df5 100644 --- a/packages/warden/src/action/workflow/pr-workflow.test.ts +++ b/packages/warden/src/action/workflow/pr-workflow.test.ts @@ -119,7 +119,9 @@ import { runSkillTask } from '../../cli/output/tasks.js'; import { fetchExistingComments, deduplicateFindings, processDuplicateActions } from '../../output/dedup.js'; import { evaluateFixAttempts } from '../fix-evaluation/index.js'; import { setFailed, writeFindingsOutput } from './base.js'; -import { runPRWorkflow } from './pr-workflow.js'; +import { runPRWorkflow, resolveWorkflowAuxiliaryOptions } from './pr-workflow.js'; +import type { WardenConfig } from '../../config/schema.js'; +import type { LoadedLayeredConfig } from '../../config/loader.js'; import { clearSkillsCache } from '../../skills/loader.js'; import { Semaphore } from '../../utils/index.js'; import { buildFindingsOutput } from '../reporting/output.js'; @@ -2264,4 +2266,45 @@ describe('runPRWorkflow', () => { expect(mockEvaluateFixAttempts).not.toHaveBeenCalled(); }); }); + + describe('resolveWorkflowAuxiliaryOptions', () => { + const layered = (config: WardenConfig, baseConfig?: WardenConfig, repoConfig?: WardenConfig): LoadedLayeredConfig => + ({ config, baseConfig, repoConfig } as LoadedLayeredConfig); + + it('inherits the global default model when no auxiliary model is set', () => { + const options = resolveWorkflowAuxiliaryOptions( + layered({ version: 1, skills: [], defaults: { model: 'litellm/gemma' } }), + ); + expect(options.model).toBe('litellm/gemma'); + }); + + it('inherits the agent model when no auxiliary model is set', () => { + const options = resolveWorkflowAuxiliaryOptions( + layered({ version: 1, skills: [], defaults: { agent: { model: 'litellm/gemma' } } }), + ); + expect(options.model).toBe('litellm/gemma'); + }); + + it('prefers an explicit auxiliary model over the inherited global model', () => { + const options = resolveWorkflowAuxiliaryOptions( + layered({ + version: 1, + skills: [], + defaults: { model: 'litellm/gemma', auxiliary: { model: 'litellm/cheap' } }, + }), + ); + expect(options.model).toBe('litellm/cheap'); + }); + + it('inherits the base layer global model when the repo layer omits it', () => { + const options = resolveWorkflowAuxiliaryOptions( + layered( + { version: 1, skills: [] }, + { version: 1, skills: [], defaults: { model: 'litellm/org' } }, + { version: 1, skills: [], defaults: { runtime: 'pi' } }, + ), + ); + expect(options.model).toBe('litellm/org'); + }); + }); }); diff --git a/packages/warden/src/action/workflow/pr-workflow.ts b/packages/warden/src/action/workflow/pr-workflow.ts index 27910844..18533664 100644 --- a/packages/warden/src/action/workflow/pr-workflow.ts +++ b/packages/warden/src/action/workflow/pr-workflow.ts @@ -175,7 +175,7 @@ function checkOptionsForPullRequest(context: EventContext): CheckOptions | undef }; } -function resolveWorkflowAuxiliaryOptions(layered: LoadedLayeredConfig): AuxiliaryWorkflowOptions { +export function resolveWorkflowAuxiliaryOptions(layered: LoadedLayeredConfig): AuxiliaryWorkflowOptions { const baseDefaults = layered.baseConfig?.defaults; const repoDefaults = layered.repoConfig?.defaults ?? layered.config.defaults; @@ -185,9 +185,17 @@ function resolveWorkflowAuxiliaryOptions(layered: LoadedLayeredConfig): Auxiliar // repo layer only fills fields the base omits. runtime: baseDefaults?.runtime ?? repoDefaults?.runtime ?? 'pi', providers: baseDefaults?.providers ?? repoDefaults?.providers, + // Inherit the global default model when no explicit auxiliary model is set, + // so workflow-scoped helper calls stay on the configured provider instead of + // falling back to a runtime default on another provider. Base-first to match + // the enforced-baseline precedence above; explicit auxiliary models win. model: emptyToUndefined(baseDefaults?.auxiliary?.model) ?? - emptyToUndefined(repoDefaults?.auxiliary?.model), + emptyToUndefined(repoDefaults?.auxiliary?.model) ?? + emptyToUndefined(baseDefaults?.agent?.model) ?? + emptyToUndefined(baseDefaults?.model) ?? + emptyToUndefined(repoDefaults?.agent?.model) ?? + emptyToUndefined(repoDefaults?.model), maxRetries: baseDefaults?.auxiliary?.maxRetries ?? baseDefaults?.auxiliaryMaxRetries ?? From b35d5933f6b9bcbdea357bc726a80d154fd284aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Tue, 30 Jun 2026 16:37:00 +0200 Subject: [PATCH 19/22] fix(action): align runtime and provider-options resolution in fix evaluation evaluateFix called getRuntime(runtimeOptions.runtime) - which defaults an omitted runtime to 'pi' - but resolved provider options with getRuntimeProviderOptions(runtimeOptions.runtime ?? 'claude', ...). On the omitted-runtime path the two disagreed: the call ran on Pi while provider options were built for Claude, so custom providers were never registered and fix evaluation silently escaped to a runtime default on another provider. Resolve one effective runtime (runtimeOptions.runtime ?? 'pi') and pass it to both getRuntime and getRuntimeProviderOptions so they can never diverge. Update the null-options test and add a test for the omitted-runtime forwarding path. Reported by Cursor Bugbot on PR #416. Co-Authored-By: Claude Opus 4.8 --- .../judge.runtime-options.test.ts | 66 ++++++++++++++++++- .../warden/src/action/fix-evaluation/judge.ts | 10 ++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts b/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts index 4ab2d07f..40913140 100644 --- a/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts +++ b/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts @@ -59,7 +59,9 @@ describe('evaluateFix runtime options', () => { ); expect(result.usedFallback).toBe(false); - expect(getRuntime).toHaveBeenCalledWith(undefined); + // Omitted runtime resolves to 'pi' (getRuntime's default), and the same + // value drives the provider-options lookup so the two never diverge. + expect(getRuntime).toHaveBeenCalledWith('pi'); expect(runAuxiliary).toHaveBeenCalledWith( expect.objectContaining({ model: undefined, @@ -134,4 +136,66 @@ describe('evaluateFix runtime options', () => { expect.objectContaining({ providerOptions }) ); }); + + it('builds provider options for the resolved runtime when runtime is omitted', async () => { + const runAuxiliary = vi.fn().mockResolvedValue({ + success: true, + data: { status: 'not_attempted', reasoning: 'No related changes' }, + usage: { inputTokens: 0, outputTokens: 0, costUSD: 0 }, + }); + const getRuntime = vi.fn(() => ({ runAuxiliary })); + const providerOptions = { providers: [{ name: 'litellm' }] }; + const getRuntimeProviderOptions = vi.fn(() => providerOptions); + + vi.doMock('../../sdk/runtimes/index.js', () => ({ + getRuntime, + getRuntimeProviderOptions, + })); + + const { evaluateFix } = await import('./judge.js'); + + const comment: ExistingComment = { + id: 1, + path: 'src/handler.ts', + line: 12, + title: 'SQL injection', + description: 'User input is concatenated into SQL', + contentHash: 'abc123', + isWarden: true, + threadId: 'thread-1', + }; + + const input: FixJudgeInput = { + comment, + changedFiles: ['src/handler.ts'], + codeBeforeFix: '12: const query = "SELECT * FROM users WHERE id = " + id;', + }; + + const context: FixJudgeContext = { + octokit: {} as Octokit, + owner: 'test-owner', + repo: 'test-repo', + baseSha: 'base123', + headSha: 'head456', + patches: new Map(), + }; + + const providers: ProvidersConfig = { + litellm: { + baseUrl: 'http://localhost:4000', + api: 'openai-completions', + models: [{ id: 'litellm/gpt-4o', name: 'gpt-4o' }], + }, + }; + + // No runtime specified: it resolves to 'pi', and the provider-options lookup + // must use that same 'pi' so custom providers are built, not dropped. + await evaluateFix(input, context, 'api-key', { providers }); + + expect(getRuntime).toHaveBeenCalledWith('pi'); + expect(getRuntimeProviderOptions).toHaveBeenCalledWith('pi', { providers }); + expect(runAuxiliary).toHaveBeenCalledWith( + expect.objectContaining({ providerOptions }) + ); + }); }); diff --git a/packages/warden/src/action/fix-evaluation/judge.ts b/packages/warden/src/action/fix-evaluation/judge.ts index d9372c6c..e5970c62 100644 --- a/packages/warden/src/action/fix-evaluation/judge.ts +++ b/packages/warden/src/action/fix-evaluation/judge.ts @@ -226,7 +226,13 @@ export async function evaluateFix( const prompt = buildPrompt(input); const executeTool = createToolExecutor(context); - const result = await getRuntime(runtimeOptions.runtime).runAuxiliary({ + // Resolve one effective runtime so getRuntime and getRuntimeProviderOptions + // never diverge. getRuntime defaults an omitted runtime to 'pi'; previously the + // provider-options lookup defaulted to 'claude', so on the omitted-runtime path + // the call ran on Pi but built Claude-shaped options and silently dropped any + // custom providers. + const runtime = runtimeOptions.runtime ?? 'pi'; + const result = await getRuntime(runtime).runAuxiliary({ task: 'fix_evaluation', agentName: input.skillName, apiKey, @@ -237,7 +243,7 @@ export async function evaluateFix( model: runtimeOptions.model, maxIterations: 5, maxRetries: runtimeOptions.maxRetries, - providerOptions: getRuntimeProviderOptions(runtimeOptions.runtime ?? 'claude', { providers: runtimeOptions.providers }), + providerOptions: getRuntimeProviderOptions(runtime, { providers: runtimeOptions.providers }), }); if (result.success) { From bff6604a4df0b8194254f5e202611c66958100e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Tue, 30 Jun 2026 16:41:39 +0200 Subject: [PATCH 20/22] fix(config): inherit base-layer custom providers into the repo layer inheritRepoLayerDefaults carried the org base runtime and verification defaults into the repo layer but not defaults.providers. A repo config that only added skills therefore lost the custom providers defined by the org base config, so its resolved triggers ran without them. Inherit defaults.providers as an execution-environment default, alongside runtime. Per-skill policy defaults (model, failOn, ignorePaths, ...) still do not cross layers, matching the existing selective-inheritance contract. Reported by Cursor Bugbot on PR #416. Co-Authored-By: Claude Opus 4.8 --- packages/warden/src/config/loader.test.ts | 41 +++++++++++++++++++++++ packages/warden/src/config/loader.ts | 8 +++++ 2 files changed, 49 insertions(+) diff --git a/packages/warden/src/config/loader.test.ts b/packages/warden/src/config/loader.test.ts index ae057115..31629872 100644 --- a/packages/warden/src/config/loader.test.ts +++ b/packages/warden/src/config/loader.test.ts @@ -941,6 +941,47 @@ describe('resolveLayeredSkillConfigs', () => { expect(resolved[1]?.runtime).toBe('pi'); }); + it('lets repo-defined skills inherit base custom providers when repo defaults omit them', () => { + const providers = { + litellm: { + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions' as const, + models: [{ id: 'gemma', name: 'gemma' }], + }, + }; + const baseConfig: WardenConfig = { + version: 1, + defaults: { + runtime: 'pi', + providers, + }, + skills: [{ + name: 'org-skill', + triggers: [{ type: 'pull_request', actions: ['opened'] }], + }], + }; + + const repoConfig: WardenConfig = { + version: 1, + skills: [{ + name: 'repo-skill', + triggers: [{ type: 'pull_request', actions: ['opened'] }], + }], + }; + + const resolved = resolveLayeredSkillConfigs({ + config: { version: 1, skills: [] }, + baseConfig, + repoConfig, + }); + + expect(resolved).toHaveLength(2); + // Both the base-layer and the repo-layer trigger carry the org custom + // providers; previously the repo layer lost them. + expect(resolved.find((t) => t.name === 'org-skill')?.providers).toEqual(providers); + expect(resolved.find((t) => t.name === 'repo-skill')?.providers).toEqual(providers); + }); + it('lets repo-defined skills inherit base verification defaults when omitted', () => { const baseConfig: WardenConfig = { version: 1, diff --git a/packages/warden/src/config/loader.ts b/packages/warden/src/config/loader.ts index 7b73d434..c3c8075b 100644 --- a/packages/warden/src/config/loader.ts +++ b/packages/warden/src/config/loader.ts @@ -180,6 +180,14 @@ function inheritRepoLayerDefaults(base?: Defaults, repo?: Defaults): Defaults | inherited.runtime = base.runtime; } + // Inherit the org base custom providers, like runtime, as an execution- + // environment default: a repo layer that only adds skills should still reach + // the providers the org defined. Per-skill policy defaults (model, failOn, + // ignorePaths, ...) intentionally do not cross layers. + if (base?.providers !== undefined && inherited.providers === undefined) { + inherited.providers = base.providers; + } + const verification = mergeNestedConfig(base?.verification, repo?.verification); if (verification) { inherited.verification = verification; From 5ef174d9544fe15ef2fab06dfa47ca6983c2d5b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Tue, 30 Jun 2026 17:28:37 +0200 Subject: [PATCH 21/22] fix(cli): bring warden build/improve in line for custom providers The generated-skill build/improve command resolved its model lanes and skipped the provider preflight differently from every other entry point, so a config that points only `defaults.model` at a custom (e.g. self-hosted) provider could silently escape to a runtime default on another provider, and a keyless remote provider failed mid-build instead of failing fast. - Synthesis model now falls back to the global default chain (synthesis -> auxiliary -> agent.model -> model -> --model -> WARDEN_MODEL) via a shared resolveDefaultModel helper, matching resolveCliDefaultSynthesisModel. - Repair model gains the same auxiliary inheritance chain instead of reading only defaults.auxiliary.model. - Add the assertCustomProviderAuth preflight (fail-fast with reporter.error) before synthesis, matching the CLI, executor, and workflow entry points. Export the resolvers and unit-test the inheritance/precedence; add an integration test that build fails fast on a keyless remote provider. Reported by Cursor Bugbot on PR #416. Co-Authored-By: Claude Opus 4.8 --- .../warden/src/cli/commands/build.test.ts | 58 ++++++++++++++++++- packages/warden/src/cli/commands/build.ts | 50 ++++++++++++++-- 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/packages/warden/src/cli/commands/build.test.ts b/packages/warden/src/cli/commands/build.test.ts index eb086428..6188d8e6 100644 --- a/packages/warden/src/cli/commands/build.test.ts +++ b/packages/warden/src/cli/commands/build.test.ts @@ -5,7 +5,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { CLIOptions } from '../args.js'; import { Reporter } from '../output/reporter.js'; import { Verbosity } from '../output/verbosity.js'; -import { runBuild, runImprove } from './build.js'; +import { runBuild, runImprove, resolveSynthesisModel, resolveRepairModel } from './build.js'; +import type { WardenConfig } from '../../config/schema.js'; import { getRepoRoot } from '../git.js'; import { buildGeneratedSkillDefinition, @@ -490,4 +491,59 @@ prompt: |- expect(buildSkillOutlineMock).not.toHaveBeenCalled(); expect(buildGeneratedSkillMock).not.toHaveBeenCalled(); }); + + it('fails fast when a remote custom provider has no key', async () => { + const reporter = createTestReporter(); + vi.spyOn(console, 'error').mockImplementation(() => undefined); + // Hermetic: ensure no ambient key resolves for this provider name. + delete process.env['WARDEN_SMOKETESTPROVIDER_API_KEY']; + delete process.env['SMOKETESTPROVIDER_API_KEY']; + writeFileSync(join(tempDir, 'warden.toml'), [ + 'version = 1', + '[defaults]', + 'runtime = "pi"', + 'model = "smoketestprovider/model"', + '[defaults.providers.smoketestprovider]', + 'baseUrl = "http://remote.example.com/v1"', + 'api = "openai-completions"', + '[[defaults.providers.smoketestprovider.models]]', + 'id = "model"', + ].join('\n'), 'utf-8'); + + const exitCode = await runBuild(createOptions(), reporter); + + expect(exitCode).toBe(1); + // Preflight fails before any synthesis work touches the provider. + expect(buildSkillOutlineMock).not.toHaveBeenCalled(); + expect(buildGeneratedSkillMock).not.toHaveBeenCalled(); + }); +}); + +describe('build model resolution', () => { + const cfg = (defaults: NonNullable): WardenConfig => + ({ version: 1, skills: [], defaults }); + + it('synthesis lane inherits the global default model when synthesis and auxiliary are unset', () => { + expect(resolveSynthesisModel(cfg({ model: 'litellm/gemma' }), createOptions())).toBe('litellm/gemma'); + }); + + it('synthesis lane inherits the agent model when unset', () => { + expect(resolveSynthesisModel(cfg({ agent: { model: 'litellm/gemma' } }), createOptions())).toBe('litellm/gemma'); + }); + + it('synthesis lane prefers an explicit synthesis model over the inherited model', () => { + expect( + resolveSynthesisModel(cfg({ model: 'litellm/gemma', synthesis: { model: 'litellm/synth' } }), createOptions()), + ).toBe('litellm/synth'); + }); + + it('repair lane inherits the global default model when auxiliary is unset', () => { + expect(resolveRepairModel(cfg({ model: 'litellm/gemma' }), createOptions())).toBe('litellm/gemma'); + }); + + it('repair lane prefers an explicit auxiliary model over the inherited model', () => { + expect( + resolveRepairModel(cfg({ model: 'litellm/gemma', auxiliary: { model: 'litellm/aux' } }), createOptions()), + ).toBe('litellm/aux'); + }); }); diff --git a/packages/warden/src/cli/commands/build.ts b/packages/warden/src/cli/commands/build.ts index 45d2bbd1..d61f66a5 100644 --- a/packages/warden/src/cli/commands/build.ts +++ b/packages/warden/src/cli/commands/build.ts @@ -36,6 +36,10 @@ import { invalidPiModelSelectorMessage, type InvalidPiModelSelector, } from '../../sdk/runtimes/model-selectors.js'; +import { + assertCustomProviderAuth, + buildPiProviderOptions, +} from '../../sdk/runtimes/custom-provider.js'; function renderHeader(args: { reporter: Reporter; @@ -157,18 +161,45 @@ function resolvePromptValue(prompt: string): string { return prompt.trim(); } -function resolveSynthesisModel( +/** + * Global default model chain shared by the build/improve lanes, mirroring the + * agent-lane resolution used elsewhere (agent.model -> model -> --model -> + * WARDEN_MODEL). Keeps the synthesis and repair lanes on the configured provider + * instead of escaping to a runtime default on another provider. + */ +function resolveDefaultModel( config: WardenConfig | undefined, options: CLIOptions, ): string | undefined { return ( - emptyToUndefined(config?.defaults?.synthesis?.model) ?? - emptyToUndefined(config?.defaults?.auxiliary?.model) ?? + emptyToUndefined(config?.defaults?.agent?.model) ?? + emptyToUndefined(config?.defaults?.model) ?? emptyToUndefined(options.model) ?? emptyToUndefined(process.env['WARDEN_MODEL']) ); } +export function resolveSynthesisModel( + config: WardenConfig | undefined, + options: CLIOptions, +): string | undefined { + return ( + emptyToUndefined(config?.defaults?.synthesis?.model) ?? + emptyToUndefined(config?.defaults?.auxiliary?.model) ?? + resolveDefaultModel(config, options) + ); +} + +export function resolveRepairModel( + config: WardenConfig | undefined, + options: CLIOptions, +): string | undefined { + return ( + emptyToUndefined(config?.defaults?.auxiliary?.model) ?? + resolveDefaultModel(config, options) + ); +} + type GeneratedSkillCommandMode = 'build' | 'improve'; function reportInvalidPiModelSelector(reporter: Reporter, invalid: InvalidPiModelSelector): void { @@ -333,7 +364,7 @@ async function runGeneratedSkillCommand( const runtimeName = config?.defaults?.runtime ?? 'pi'; const providers = config?.defaults?.providers; const model = resolveSynthesisModel(config, options); - const repairModel = emptyToUndefined(config?.defaults?.auxiliary?.model); + const repairModel = resolveRepairModel(config, options); const maxRetries = config?.defaults?.auxiliary?.maxRetries ?? config?.defaults?.auxiliaryMaxRetries; const invalidModelSelector = findInvalidPiModelSelector([{ runtime: runtimeName, @@ -344,6 +375,17 @@ async function runGeneratedSkillCommand( reportInvalidPiModelSelector(reporter, invalidModelSelector); return 1; } + // Fail fast when a remote custom provider has no resolvable key, matching the + // CLI, executor, and workflow entry points. Otherwise build would call the + // provider and fail mid-synthesis instead. + if (runtimeName === 'pi') { + try { + assertCustomProviderAuth(buildPiProviderOptions(providers, process.env)); + } catch (error) { + reporter.error(error instanceof Error ? error.message : String(error)); + return 1; + } + } const runtime = getRuntime(runtimeName); if (!options.json) { From e31e6fec9145f5e84aa7fee0af11facde042bb0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Hauser?= Date: Tue, 30 Jun 2026 18:46:15 +0200 Subject: [PATCH 22/22] fix(skill-builder): forward custom providers to the fallback JSON repair In runStructuredSkillBuilderAgent, when structured output fails validation and the primary repairStructuredSkillBuilderOutput path also fails, the secondary parseJsonFromOutput repair call omitted `providers`. The primary path forwards them, so on Pi with a self-hosted model the fallback repair ran without the registered custom providers and could fail or hit the wrong backend. Pass providers through to the fallback repair options (JsonOutputRepairOptions already supports and forwards them). Add a test covering the fallback path. Reported by Cursor Bugbot on PR #416. Co-Authored-By: Claude Opus 4.8 --- .../warden/src/skill-builder/agentic.test.ts | 72 +++++++++++++++++++ packages/warden/src/skill-builder/agentic.ts | 4 ++ 2 files changed, 76 insertions(+) create mode 100644 packages/warden/src/skill-builder/agentic.test.ts diff --git a/packages/warden/src/skill-builder/agentic.test.ts b/packages/warden/src/skill-builder/agentic.test.ts new file mode 100644 index 00000000..864fe308 --- /dev/null +++ b/packages/warden/src/skill-builder/agentic.test.ts @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; + +vi.mock('../sdk/json-output.js', () => ({ + parseJsonFromOutput: vi.fn(), +})); + +import { parseJsonFromOutput } from '../sdk/json-output.js'; +import { runStructuredSkillBuilderAgent } from './agentic.js'; +import { emptyUsage } from '../sdk/usage.js'; +import type { Runtime } from '../sdk/runtimes/index.js'; +import type { ProvidersConfig } from '../config/schema.js'; + +const parseJsonFromOutputMock = vi.mocked(parseJsonFromOutput); + +const schema = z.object({ ok: z.boolean() }); + +describe('runStructuredSkillBuilderAgent provider forwarding', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('forwards custom providers to the fallback auxiliary JSON repair', async () => { + const providers: ProvidersConfig = { + litellm: { + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions', + models: [{ id: 'gemma', name: 'gemma' }], + }, + }; + + // Main agent run succeeds with unparseable text; the structured-repair agent + // run then fails (no result), forcing the fallback auxiliary repair path. + const runSkill = vi + .fn() + .mockResolvedValueOnce({ + result: { status: 'success', text: 'NOT JSON', errors: [], usage: emptyUsage() }, + }) + .mockResolvedValue({}); + const runtime = { + name: 'pi', + runSkill, + runAuxiliary: vi.fn(), + runSynthesis: vi.fn(), + } as unknown as Runtime; + + // Primary parse fails; the fallback auxiliary repair succeeds. + parseJsonFromOutputMock + .mockResolvedValueOnce({ success: false, error: 'invalid_json: nope' }) + .mockResolvedValue({ success: true, data: { ok: true }, json: '{"ok":true}', repaired: true }); + + const result = await runStructuredSkillBuilderAgent({ + runtime, + repoPath: '/repo', + skillName: 'security', + systemPrompt: 'sys', + userPrompt: 'user', + schema, + providers, + repair: { apiKey: 'repair-key', model: 'repair-model', maxRetries: 2 }, + }); + + expect(result.data).toEqual({ ok: true }); + // The fallback repair (second parse call) must carry the custom providers. + expect(parseJsonFromOutputMock).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + repair: expect.objectContaining({ providers }), + }), + ); + }); +}); diff --git a/packages/warden/src/skill-builder/agentic.ts b/packages/warden/src/skill-builder/agentic.ts index 448a6e90..e0f34094 100644 --- a/packages/warden/src/skill-builder/agentic.ts +++ b/packages/warden/src/skill-builder/agentic.ts @@ -285,6 +285,10 @@ export async function runStructuredSkillBuilderAgent(args: { apiKey: args.repair.apiKey, model: args.repair.model, maxRetries: args.repair.maxRetries, + // Forward custom providers like the primary repair path does, so the + // fallback auxiliary repair stays on the configured (e.g. self-hosted) + // provider instead of the runtime default. + providers: args.providers, }, }); if (auxiliaryRepair.usage) {