diff --git a/docs/superpowers/plans/2026-06-26-warden-custom-provider-litellm.md b/docs/superpowers/plans/2026-06-26-warden-custom-provider-litellm.md new file mode 100644 index 00000000..622f7067 --- /dev/null +++ b/docs/superpowers/plans/2026-06-26-warden-custom-provider-litellm.md @@ -0,0 +1,979 @@ +# Custom Pi Provider (LiteLLM / self-hosted) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let users register a self-hosted, OpenAI-compatible endpoint (e.g. LiteLLM) as a named provider in `warden.toml` and target its models with the existing `provider/model` selector, across all model lanes. + +**Architecture:** Add a `[defaults.providers.]` config block, normalize it (with sensible per-model defaults) into a Pi provider-options object whose API key is resolved from the environment, and register it on Pi's `ModelRegistry` via `registerProvider` inside `runPiPrompt` (the single choke point all Pi lanes flow through). Thread the raw config from the resolved trigger config to every runtime call site alongside the existing `runtime`/`auxiliaryModel` fields. + +**Tech Stack:** TypeScript (strict, ESM), Zod v4, Vitest, `@earendil-works/pi-coding-agent@0.78.0` (`ModelRegistry.registerProvider`), `@earendil-works/pi-ai@0.78.0` (`Api = "openai-completions"`). + +## Global Constraints + +- TypeScript strict mode; use `export type` for type-only exports (Bun compatibility). +- Zod for all runtime validation; new schemas live in `config/schema.ts`. +- No secrets in `warden.toml`. API keys resolve from env only. +- Co-locate tests (`foo.ts` → `foo.test.ts`). Prefer integration over unit; mock the HTTP/SDK boundary, never call a live endpoint. +- Verify with `pnpm lint && pnpm build && pnpm test` (run targeted vitest during tasks). +- Commit messages use the repo convention and end with `Co-Authored-By: Claude Opus 4.8 `. +- Only `openai-completions` is a valid `api` value in v1. The field exists so `anthropic-messages` can be added later. + +--- + +## File Structure + +- **Create** `packages/warden/src/sdk/runtimes/custom-provider.ts` — normalization, env key resolution, auth assertion. Pure functions, no I/O beyond reading a passed-in env object. +- **Create** `packages/warden/src/sdk/runtimes/custom-provider.test.ts` — unit tests for the above. +- **Modify** `packages/warden/src/config/schema.ts` — add provider schemas + `DefaultsSchema.providers`. +- **Modify** `packages/warden/src/config/schema.test.ts` (or `config/loader.test.ts` if no schema test exists) — schema accept/reject tests. +- **Modify** `packages/warden/src/sdk/runtimes/index.ts` — extend `getRuntimeProviderOptions` for `pi`. +- **Modify** `packages/warden/src/sdk/runtimes/types.ts` — add `providerOptions` to auxiliary/synthesis requests. +- **Modify** `packages/warden/src/sdk/runtimes/pi.ts` — register custom providers in `runPiPrompt`; read `providerOptions` in `runSkill`/`runStructured`. +- **Modify** `packages/warden/src/sdk/runtimes/pi.test.ts` — registration integration test. +- **Modify** `packages/warden/src/sdk/types.ts` — `SkillRunnerOptions.providers`. +- **Modify** `packages/warden/src/sdk/analyze.ts`, `verify.ts`, `extract.ts` — thread `providers` into agent + extraction runtime calls. +- **Modify** `packages/warden/src/output/dedup.ts`, `action/fix-evaluation/judge.ts`, `sdk/json-output.ts` — thread `providers` into auxiliary/synthesis calls. +- **Modify** `packages/warden/src/config/loader.ts` — `ResolvedTrigger.providers` + assignment. +- **Modify** runner entry points: `action/triggers/executor.ts`, `action/workflow/schedule.ts`, `action/workflow/pr-workflow.ts`, `action/review/poster.ts`, `cli/main.ts` — copy `providers` alongside `auxiliaryModel`. +- **Modify** `packages/docs/src/content/docs/config/models.mdx` — docs. + +--- + +## Task 1: Provider config schema + +**Files:** +- Modify: `packages/warden/src/config/schema.ts` (add near `DefaultsSchema`, before line 220) +- Test: `packages/warden/src/config/loader.test.ts` (add a `describe` block; this file already imports the schema/loader) + +**Interfaces:** +- Produces: `ProviderModelConfigSchema`, `ProviderConfigSchema`, `ProvidersConfigSchema`; types `ProviderModelConfig`, `ProviderConfig`, `ProvidersConfig`; new optional `DefaultsSchema.providers` field of type `ProvidersConfig`. + +- [ ] **Step 1: Write the failing test** + +Add to `packages/warden/src/config/loader.test.ts` (top-level, near other `describe`s). It parses TOML through the existing loader the same way other tests in this file do — check the file head for the existing `parseConfig`/`loadConfig` import and reuse it. If the file exposes `WardenConfigSchema` parsing helpers, prefer those. The assertions: + +```ts +import { WardenConfigSchema } from './schema.js'; + +describe('providers config', () => { + const base = { version: 1 as const, skills: [] }; + + it('accepts a valid custom provider', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { + providers: { + litellm: { + baseUrl: 'http://localhost:4000/v1', + models: [{ id: 'my-model' }], + }, + }, + }, + }); + expect(result.success).toBe(true); + if (result.success) { + const p = result.data.defaults?.providers?.['litellm']; + expect(p?.api).toBe('openai-completions'); // default applied + expect(p?.models[0]?.id).toBe('my-model'); + } + }); + + it('rejects a non-URL baseUrl', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { litellm: { baseUrl: 'not a url', models: [{ id: 'm' }] } } }, + }); + expect(result.success).toBe(false); + }); + + it('rejects an unsupported api value', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { x: { baseUrl: 'http://h/v1', api: 'cohere', models: [{ id: 'm' }] } } }, + }); + expect(result.success).toBe(false); + }); + + it('rejects an empty models array', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { x: { baseUrl: 'http://h/v1', models: [] } } }, + }); + expect(result.success).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- config/loader.test.ts -t "providers config"` +Expected: FAIL (TOML parses but `providers` is stripped/unknown, so `api` default assertion fails; reject cases pass-through as success). + +- [ ] **Step 3: Add the schemas** + +In `packages/warden/src/config/schema.ts`, insert before `DefaultsSchema` (currently line 220): + +```ts +// Per-model definition for a custom provider. Only `id` is required; Warden +// fills the remaining fields Pi requires with sensible defaults. +export const ProviderModelConfigSchema = z.object({ + /** Model id as exposed by the endpoint (e.g. the LiteLLM model name). */ + id: z.string().min(1), + /** Display name. Defaults to `id`. */ + name: z.string().min(1).optional(), + /** Whether the model supports reasoning/thinking. Default: false. */ + reasoning: z.boolean().optional(), + /** Accepted input modalities. Default: ["text"]. */ + input: z.array(z.enum(['text', 'image'])).min(1).optional(), + /** Context window in tokens. Default: 128000. */ + contextWindow: z.number().int().positive().optional(), + /** Max output tokens. Default: 8192. */ + maxTokens: z.number().int().positive().optional(), + /** Per-token cost (USD per token). Default: all zeros. */ + cost: z.object({ + input: z.number().nonnegative(), + output: z.number().nonnegative(), + cacheRead: z.number().nonnegative(), + cacheWrite: z.number().nonnegative(), + }).optional(), +}).strict(); +export type ProviderModelConfig = z.infer; + +// A custom, self-hosted or gateway provider (e.g. LiteLLM). OpenAI-compatible only in v1. +export const ProviderConfigSchema = z.object({ + /** OpenAI-compatible base URL, typically ending in /v1. */ + baseUrl: z.string().url(), + /** Wire protocol. Only "openai-completions" is supported today. */ + api: z.enum(['openai-completions']).default('openai-completions'), + /** Extra HTTP headers sent on every request. */ + headers: z.record(z.string(), z.string()).optional(), + /** Env var holding the API key. Defaults to WARDEN__API_KEY then _API_KEY. */ + apiKeyEnv: z.string().min(1).optional(), + /** Models this provider exposes. At least one is required. */ + models: z.array(ProviderModelConfigSchema).min(1), +}).strict(); +export type ProviderConfig = z.infer; + +// Map of provider name -> provider config. The name becomes the model-selector prefix. +export const ProvidersConfigSchema = z.record(z.string().min(1), ProviderConfigSchema); +export type ProvidersConfig = z.infer; +``` + +Then add to `DefaultsSchema` (after the `scan` field, around line 257): + +```ts + /** Custom OpenAI-compatible providers (e.g. self-hosted LiteLLM). Keyed by provider name. */ + providers: ProvidersConfigSchema.optional(), +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm --filter @sentry/warden test -- config/loader.test.ts -t "providers config"` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/config/schema.ts packages/warden/src/config/loader.test.ts +git commit -m "feat(config): add custom provider schema for self-hosted LLMs + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 2: Provider normalization + env key resolution + +**Files:** +- Create: `packages/warden/src/sdk/runtimes/custom-provider.ts` +- Test: `packages/warden/src/sdk/runtimes/custom-provider.test.ts` + +**Interfaces:** +- Consumes: `ProvidersConfig`, `ProviderConfig`, `ProviderModelConfig` from `../../config/schema.js`. +- Produces: + - `interface PiProviderModel { id; name; api: 'openai-completions'; reasoning: boolean; input: ('text'|'image')[]; cost: { input; output; cacheRead; cacheWrite }; contextWindow: number; maxTokens: number }` + - `interface PiProvider { name: string; baseUrl: string; api: 'openai-completions'; headers?: Record; apiKey?: string; models: PiProviderModel[] }` + - `type PiProviderOptions = { providers: PiProvider[] } | undefined` + - `function resolveProviderApiKey(name: string, apiKeyEnv: string | undefined, env: NodeJS.ProcessEnv): string | undefined` + - `function buildPiProviderOptions(providers: ProvidersConfig | undefined, env: NodeJS.ProcessEnv): PiProviderOptions` + - `function isLoopbackBaseUrl(baseUrl: string): boolean` + - `function assertCustomProviderAuth(options: PiProviderOptions): void` (throws `Error` naming the provider + expected env var) + +- [ ] **Step 1: Write the failing test** + +Create `packages/warden/src/sdk/runtimes/custom-provider.test.ts`: + +```ts +import { describe, it, expect } from 'vitest'; +import { + buildPiProviderOptions, + resolveProviderApiKey, + isLoopbackBaseUrl, + assertCustomProviderAuth, +} from './custom-provider.js'; +import type { ProvidersConfig } from '../../config/schema.js'; + +const providers: ProvidersConfig = { + litellm: { baseUrl: 'https://gw.example.com/v1', api: 'openai-completions', models: [{ id: 'my-model' }] }, +}; + +describe('resolveProviderApiKey', () => { + it('prefers apiKeyEnv when set', () => { + expect(resolveProviderApiKey('litellm', 'MY_KEY', { MY_KEY: 'k1', WARDEN_LITELLM_API_KEY: 'k2' })).toBe('k1'); + }); + it('falls back to WARDEN__API_KEY then _API_KEY', () => { + expect(resolveProviderApiKey('litellm', undefined, { WARDEN_LITELLM_API_KEY: 'k2' })).toBe('k2'); + expect(resolveProviderApiKey('litellm', undefined, { LITELLM_API_KEY: 'k3' })).toBe('k3'); + }); + it('returns undefined when nothing is set', () => { + expect(resolveProviderApiKey('litellm', undefined, {})).toBeUndefined(); + }); +}); + +describe('buildPiProviderOptions', () => { + it('returns undefined for empty input', () => { + expect(buildPiProviderOptions(undefined, {})).toBeUndefined(); + expect(buildPiProviderOptions({}, {})).toBeUndefined(); + }); + it('applies model defaults and resolves the key', () => { + const built = buildPiProviderOptions(providers, { WARDEN_LITELLM_API_KEY: 'k2' }); + const p = built?.providers[0]; + expect(p?.name).toBe('litellm'); + expect(p?.apiKey).toBe('k2'); + const m = p?.models[0]; + expect(m).toMatchObject({ + id: 'my-model', + name: 'my-model', + api: 'openai-completions', + reasoning: false, + input: ['text'], + contextWindow: 128000, + maxTokens: 8192, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }); + }); +}); + +describe('isLoopbackBaseUrl', () => { + it('recognizes loopback hosts', () => { + expect(isLoopbackBaseUrl('http://localhost:4000/v1')).toBe(true); + expect(isLoopbackBaseUrl('http://127.0.0.1:4000')).toBe(true); + expect(isLoopbackBaseUrl('https://gw.example.com/v1')).toBe(false); + }); +}); + +describe('assertCustomProviderAuth', () => { + it('throws for a non-loopback provider without a key', () => { + const built = buildPiProviderOptions(providers, {}); + expect(() => assertCustomProviderAuth(built)).toThrow(/litellm/); + }); + it('passes for a loopback provider without a key', () => { + const built = buildPiProviderOptions( + { local: { baseUrl: 'http://localhost:4000/v1', api: 'openai-completions', models: [{ id: 'm' }] } }, + {}, + ); + expect(() => assertCustomProviderAuth(built)).not.toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- runtimes/custom-provider.test.ts` +Expected: FAIL with module-not-found / export errors. + +- [ ] **Step 3: Implement the module** + +Create `packages/warden/src/sdk/runtimes/custom-provider.ts`: + +```ts +/** + * Normalization and auth resolution for custom OpenAI-compatible providers + * (e.g. self-hosted LiteLLM). Pure functions over a passed-in env object so + * they are trivially testable and never read process.env implicitly. + */ +import type { ProvidersConfig } from '../../config/schema.js'; + +const DEFAULT_CONTEXT_WINDOW = 128_000; +const DEFAULT_MAX_TOKENS = 8_192; +const DEFAULT_COST = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } as const; + +export interface PiProviderModel { + id: string; + name: string; + api: 'openai-completions'; + reasoning: boolean; + input: ('text' | 'image')[]; + cost: { input: number; output: number; cacheRead: number; cacheWrite: number }; + contextWindow: number; + maxTokens: number; +} + +export interface PiProvider { + name: string; + baseUrl: string; + api: 'openai-completions'; + headers?: Record; + apiKey?: string; + models: PiProviderModel[]; +} + +export type PiProviderOptions = { providers: PiProvider[] } | undefined; + +/** Resolve a provider API key from the environment. apiKeyEnv wins, then conventional names. */ +export function resolveProviderApiKey( + name: string, + apiKeyEnv: string | undefined, + env: NodeJS.ProcessEnv, +): string | undefined { + const upper = name.toUpperCase().replace(/[^A-Z0-9]/g, '_'); + const candidates = apiKeyEnv ? [apiKeyEnv] : [`WARDEN_${upper}_API_KEY`, `${upper}_API_KEY`]; + for (const candidate of candidates) { + const value = env[candidate]; + if (value) return value; + } + return undefined; +} + +/** Normalize warden.toml providers into Pi registerProvider input with defaults + resolved keys. */ +export function buildPiProviderOptions( + providers: ProvidersConfig | undefined, + env: NodeJS.ProcessEnv, +): PiProviderOptions { + if (!providers) return undefined; + const entries = Object.entries(providers); + if (entries.length === 0) return undefined; + + const built: PiProvider[] = entries.map(([name, config]) => ({ + name, + baseUrl: config.baseUrl, + api: config.api, + ...(config.headers ? { headers: config.headers } : {}), + apiKey: resolveProviderApiKey(name, config.apiKeyEnv, env), + models: config.models.map((model) => ({ + id: model.id, + name: model.name ?? model.id, + api: config.api, + reasoning: model.reasoning ?? false, + input: model.input ?? ['text'], + cost: model.cost ?? { ...DEFAULT_COST }, + contextWindow: model.contextWindow ?? DEFAULT_CONTEXT_WINDOW, + maxTokens: model.maxTokens ?? DEFAULT_MAX_TOKENS, + })), + })); + + return { providers: built }; +} + +/** True when the base URL points at a loopback host (unauthenticated runs allowed). */ +export function isLoopbackBaseUrl(baseUrl: string): boolean { + try { + const host = new URL(baseUrl).hostname; + return host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '[::1]'; + } catch { + return false; + } +} + +/** Fail fast when a non-loopback provider has no resolvable API key. */ +export function assertCustomProviderAuth(options: PiProviderOptions): void { + if (!options) return; + for (const provider of options.providers) { + if (!provider.apiKey && !isLoopbackBaseUrl(provider.baseUrl)) { + throw new Error( + `Custom provider "${provider.name}" has no API key. ` + + `Set WARDEN_${provider.name.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_API_KEY ` + + `(or the configured apiKeyEnv), or use a localhost baseUrl for an unauthenticated endpoint.`, + ); + } + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm --filter @sentry/warden test -- runtimes/custom-provider.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/sdk/runtimes/custom-provider.ts packages/warden/src/sdk/runtimes/custom-provider.test.ts +git commit -m "feat(runtime): normalize custom providers and resolve keys from env + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 3: Wire provider options into `getRuntimeProviderOptions` + +**Files:** +- Modify: `packages/warden/src/sdk/runtimes/index.ts:37-53` +- Modify: `packages/warden/src/sdk/runtimes/types.ts` (add `providerOptions` to auxiliary/synthesis requests) +- Test: `packages/warden/src/sdk/runtimes/index.test.ts` (exists) + +**Interfaces:** +- Consumes: `buildPiProviderOptions`, `PiProviderOptions` from `./custom-provider.js`; `ProvidersConfig` from `../../config/schema.js`. +- Produces: `RuntimeProviderOptionsInput` gains `providers?: ProvidersConfig`; `getRuntimeProviderOptions('pi', { providers })` returns `PiProviderOptions`. `AuxiliaryRunRequestBase`/`SynthesisRunRequest` gain `providerOptions?: unknown`. + +- [ ] **Step 1: Write the failing test** + +Add to `packages/warden/src/sdk/runtimes/index.test.ts`: + +```ts +import { getRuntimeProviderOptions } from './index.js'; + +describe('getRuntimeProviderOptions pi providers', () => { + it('builds pi provider options from config', () => { + const result = getRuntimeProviderOptions('pi', { + providers: { litellm: { baseUrl: 'http://localhost:4000/v1', api: 'openai-completions', models: [{ id: 'm' }] } }, + }) as { providers: { name: string; models: { id: string }[] }[] } | undefined; + expect(result?.providers[0]?.name).toBe('litellm'); + expect(result?.providers[0]?.models[0]?.id).toBe('m'); + }); + + it('returns undefined for pi without providers', () => { + expect(getRuntimeProviderOptions('pi', {})).toBeUndefined(); + }); + + it('still returns the claude executable path', () => { + expect(getRuntimeProviderOptions('claude', { pathToClaudeCodeExecutable: '/bin/claude' })) + .toEqual({ pathToClaudeCodeExecutable: '/bin/claude' }); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- runtimes/index.test.ts -t "pi providers"` +Expected: FAIL (pi branch currently returns undefined even with providers). + +- [ ] **Step 3: Implement** + +In `packages/warden/src/sdk/runtimes/index.ts`, replace the `RuntimeProviderOptionsInput` interface and `getRuntimeProviderOptions` body (lines 37-53): + +```ts +import { buildPiProviderOptions } from './custom-provider.js'; +import type { ProvidersConfig } from '../../config/schema.js'; + +export interface RuntimeProviderOptionsInput { + pathToClaudeCodeExecutable?: string; + providers?: ProvidersConfig; +} + +/** + * Build provider-specific runtime options at the runtime boundary. + */ +export function getRuntimeProviderOptions( + name: RuntimeName, + options: RuntimeProviderOptionsInput +): unknown { + if (name === 'claude') { + return { pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable }; + } + + if (name === 'pi') { + return buildPiProviderOptions(options.providers, process.env); + } + + return undefined; +} +``` + +Add the `import` lines to the top of the file with the other imports (do not duplicate the existing `claudeRuntime`/`piRuntime` imports). + +In `packages/warden/src/sdk/runtimes/types.ts`, add `providerOptions?: unknown;` to `AuxiliaryRunRequestBase` (after `maxRetries?` at line 112) and to `SynthesisRunRequest` (after `maxRetries?` at line 139), each with the doc comment: + +```ts + /** Provider-specific settings consumed only by the selected runtime adapter. */ + providerOptions?: unknown; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm --filter @sentry/warden test -- runtimes/index.test.ts -t "pi providers"` +Expected: PASS (3 tests). + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/sdk/runtimes/index.ts packages/warden/src/sdk/runtimes/types.ts packages/warden/src/sdk/runtimes/index.test.ts +git commit -m "feat(runtime): expose custom providers via getRuntimeProviderOptions + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 4: Register custom providers in the Pi adapter + +**Files:** +- Modify: `packages/warden/src/sdk/runtimes/pi.ts` (PiPromptOptions ~98-117; `runPiPrompt` registry creation ~346-348; `runSkill` request destructure ~751-796; `runStructured` call ~683-697) +- Test: `packages/warden/src/sdk/runtimes/pi.test.ts` + +**Interfaces:** +- Consumes: `PiProviderOptions` from `./custom-provider.js`; `ModelRegistry.registerProvider` and `AuthStorage.setRuntimeApiKey` from the Pi SDK. +- Produces: `runSkill`/`runStructured` honor `request.providerOptions` of shape `PiProviderOptions`; registered providers are resolvable by `resolvePiModel`. + +- [ ] **Step 1: Write the failing test** + +`pi.test.ts` mocks the Pi SDK. Inspect the existing mock setup at the top of `packages/warden/src/sdk/runtimes/pi.test.ts` and reuse it. The new test asserts that when `providerOptions` carries a provider, `registerProvider` is called with the normalized config before the model is resolved. Add: + +```ts +it('registers custom providers from providerOptions before resolving the model', async () => { + // registerProvider is a vi.fn() on the mocked ModelRegistry instance — see the + // existing mock factory in this file and extend it to capture registerProvider calls. + const calls = capturedRegisterProviderCalls; // provided by the mock factory (see Step 3) + calls.length = 0; + + await piRuntime.runAuxiliary({ + task: 'extraction', + apiKey: undefined, + prompt: 'hi', + schema: z.object({ ok: z.boolean() }), + model: 'litellm/my-model', + providerOptions: { + providers: [{ + name: 'litellm', + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions', + apiKey: 'k', + models: [{ + id: 'my-model', name: 'my-model', api: 'openai-completions', reasoning: false, + input: ['text'], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, maxTokens: 8192, + }], + }], + }, + }); + + expect(calls[0]?.[0]).toBe('litellm'); + expect(calls[0]?.[1]).toMatchObject({ baseUrl: 'http://localhost:4000/v1', apiKey: 'k' }); +}); +``` + +If the existing mock does not yet expose `registerProvider`/a capture array, that is part of Step 3. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- runtimes/pi.test.ts -t "registers custom providers"` +Expected: FAIL (registerProvider never called / mock missing). + +- [ ] **Step 3: Implement** + +In `packages/warden/src/sdk/runtimes/pi.ts`: + +(a) Add the import near the other local imports: + +```ts +import type { PiProviderOptions } from './custom-provider.js'; +``` + +(b) Add to `PiPromptOptions` (after `legacyAnthropicApiKey?` ~line 104): + +```ts + /** Custom providers to register on the model registry before resolving the model. */ + customProviders?: PiProviderOptions; +``` + +(c) Add a helper near `createAuthStorage` (after line 151): + +```ts +function registerCustomProviders( + modelRegistry: ModelRegistry, + authStorage: AuthStorage, + customProviders: PiProviderOptions, +): void { + if (!customProviders) return; + for (const provider of customProviders.providers) { + modelRegistry.registerProvider(provider.name, { + baseUrl: provider.baseUrl, + api: provider.api, + ...(provider.headers ? { headers: provider.headers } : {}), + ...(provider.apiKey ? { apiKey: provider.apiKey } : {}), + models: provider.models, + }); + if (provider.apiKey) { + authStorage.setRuntimeApiKey(provider.name, provider.apiKey); + } + } +} +``` + +(d) In `runPiPrompt`, right after `const modelRegistry = ModelRegistry.create(authStorage);` (line 347) and before `const model = resolvePiModel(...)` (line 348): + +```ts + registerCustomProviders(modelRegistry, authStorage, options.customProviders); +``` + +(e) In `runSkill` destructure `providerOptions` from `request` (add to the destructure block ~751-761), then pass it through to `runPiPrompt` (in the `runPiPrompt({ ... })` call ~783-796): + +```ts + customProviders: request.providerOptions as PiProviderOptions, +``` + +(f) In `runStructured`, the request object already spreads `...request`. Pass it into the `runPiPrompt` call (~683-697): + +```ts + customProviders: request.providerOptions as PiProviderOptions, +``` + +(`runStructured`'s `request` param type is the inline object; add `providerOptions?: unknown;` to it alongside the other fields ~643-658.) + +(g) Update the mock factory in `pi.test.ts` so the mocked `ModelRegistry` instance has `registerProvider: vi.fn((...args) => capturedRegisterProviderCalls.push(args))` and export/declare `capturedRegisterProviderCalls` at module scope, and the mocked `AuthStorage` has `setRuntimeApiKey: vi.fn()`. Mirror how the existing mock exposes `find`/`create`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm --filter @sentry/warden test -- runtimes/pi.test.ts` +Expected: PASS (new test + existing tests still green). + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/sdk/runtimes/pi.ts packages/warden/src/sdk/runtimes/pi.test.ts +git commit -m "feat(runtime): register custom providers in the pi adapter + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 5: Thread `providers` through the SDK runner + all runtime call sites + +**Files:** +- Modify: `packages/warden/src/sdk/types.ts` (`SkillRunnerOptions`, after line 116 `runtime?`) +- Modify: `packages/warden/src/sdk/analyze.ts:397-399` (agent call) and the extraction call site +- Modify: `packages/warden/src/sdk/verify.ts:268-270` (and `VerifyFindingsOptions`) +- Modify: `packages/warden/src/sdk/extract.ts:29-35` (`AuxiliaryCallOptions`) and `:230-238` +- Modify: `packages/warden/src/output/dedup.ts:616` and `:919` +- Modify: `packages/warden/src/action/fix-evaluation/judge.ts:208-229` +- Modify: `packages/warden/src/sdk/json-output.ts:72` +- Test: `packages/warden/src/sdk/extract.test.ts` (assert providerOptions is forwarded) + +**Interfaces:** +- Consumes: `getRuntimeProviderOptions` (Task 3); `ProvidersConfig` from `config/schema.js`. +- Produces: `SkillRunnerOptions.providers?: ProvidersConfig`; `AuxiliaryCallOptions.providers?: ProvidersConfig`; `VerifyFindingsOptions.providers?: ProvidersConfig`; every Pi-capable runtime request carries `providerOptions` derived from `providers`. + +- [ ] **Step 1: Write the failing test** + +In `packages/warden/src/sdk/extract.test.ts`, add a test that stubs the runtime and asserts `providerOptions` is forwarded. Reuse the file's existing runtime mock (look for where `getRuntime`/`runAuxiliary` is mocked). The assertion: + +```ts +it('forwards providerOptions built from providers to the auxiliary runtime', async () => { + // Arrange: runAuxiliary mock captures its request; runtime = 'pi'. + await extractFindingsWithLLM(rawTextWithFindings, { + runtime: 'pi', + providers: { litellm: { baseUrl: 'http://localhost:4000/v1', api: 'openai-completions', models: [{ id: 'm' }] } }, + }); + const req = runAuxiliaryMock.mock.calls[0][0]; + expect(req.providerOptions).toBeDefined(); + expect(req.providerOptions.providers[0].name).toBe('litellm'); +}); +``` + +(Adapt `rawTextWithFindings`/option shape to the existing test helpers in the file.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- extract.test.ts -t "forwards providerOptions"` +Expected: FAIL (`providers` not a known option; `providerOptions` undefined on the request). + +- [ ] **Step 3: Implement the threading** + +Make these edits (each mirrors the existing `runtime`/`auxiliaryModel` handling): + +1. `sdk/types.ts` — in `SkillRunnerOptions`, after `runtime?` (line 116): +```ts + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; +``` +Add the import: `import type { ProvidersConfig } from '../config/schema.js';` (extend the existing `config/schema.js` import on line 3). + +2. `sdk/extract.ts` — in `AuxiliaryCallOptions` (line 29-35) add: +```ts + providers?: ProvidersConfig; +``` +Import `ProvidersConfig` and `getRuntimeProviderOptions` (the latter is already re-exported via runner; import from `./runtimes/index.js`). In the `runAuxiliary` call (line 230), add: +```ts + providerOptions: getRuntimeProviderOptions(runtimeName, { providers: options.providers }), +``` + +3. `sdk/analyze.ts` — the extraction call delegates to `extractFindingsWithLLM`; ensure the options passed there include `providers: options.providers`. Find the `extractFindingsWithLLM(` / extraction-options construction in this file and add `providers: options.providers`. Also the agent `getRuntimeProviderOptions` call (line 397) becomes: +```ts + providerOptions: getRuntimeProviderOptions(runtimeName, { + pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable, + providers: options.providers, + }), +``` + +4. `sdk/verify.ts` — add `providers?: ProvidersConfig` to `VerifyFindingsOptions`; line 268 call becomes: +```ts + providerOptions: getRuntimeProviderOptions(runtimeName, { + pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable, + providers: options.providers, + }), +``` + +5. `output/dedup.ts` — add `providers?: ProvidersConfig` to the dedup options type; both `runAuxiliary` calls (lines 616, 919) add: +```ts + providerOptions: getRuntimeProviderOptions(options.runtime ?? 'claude', { providers: options.providers }), +``` + +6. `action/fix-evaluation/judge.ts` — add `providers?: ProvidersConfig` to `runtimeOptions`; the `runAuxiliary` call (line 226) adds: +```ts + providerOptions: getRuntimeProviderOptions(runtimeOptions.runtime, { providers: runtimeOptions.providers }), +``` + +7. `sdk/json-output.ts` — the repair path (line 72) builds a runtime request; add `providerOptions: getRuntimeProviderOptions(repair.runtimeName ?? 'claude', { providers: repair.providers })` to that request and `providers?: ProvidersConfig` to the repair options type. + +For each, import `getRuntimeProviderOptions` from `./runtimes/index.js` (or `../sdk/runtimes/index.js` for non-sdk files) and `ProvidersConfig` from the appropriate `config/schema.js` relative path. + +- [ ] **Step 4: Run tests** + +Run: `pnpm --filter @sentry/warden test -- extract.test.ts -t "forwards providerOptions"` → PASS. +Then `pnpm --filter @sentry/warden typecheck` → no errors (catches any missed `providers` field or import). + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/sdk packages/warden/src/output/dedup.ts packages/warden/src/action/fix-evaluation/judge.ts +git commit -m "feat(runtime): thread custom providers through all model lanes + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 6: Resolve `providers` from config and pass it to every runner entry point + +**Files:** +- Modify: `packages/warden/src/config/loader.ts` (`ResolvedTrigger` ~400, `resolveSkillConfigs` ~495 + both push branches ~546-556 and ~581-592) +- Modify: `packages/warden/src/action/triggers/executor.ts:189-192` +- Modify: `packages/warden/src/action/workflow/schedule.ts:216-219` +- Modify: `packages/warden/src/action/workflow/pr-workflow.ts:184` + auxiliary/fix-eval option construction (~313, ~566, ~635, ~685) +- Modify: `packages/warden/src/action/review/poster.ts:255-315` (consolidate + dedup option construction) +- Modify: `packages/warden/src/cli/main.ts:1100-1130, 1153-1171` (runner options + per-trigger options) +- Test: `packages/warden/src/config/loader.test.ts` + +**Interfaces:** +- Consumes: `WardenConfig.defaults.providers` (Task 1). +- Produces: `ResolvedTrigger.providers?: ProvidersConfig`, populated from `defaults.providers`; carried into `SkillRunnerOptions.providers` and every auxiliary/synthesis options object. + +- [ ] **Step 1: Write the failing test** + +In `packages/warden/src/config/loader.test.ts`, add: + +```ts +it('propagates defaults.providers onto every resolved trigger', () => { + const config = WardenConfigSchema.parse({ + version: 1, + defaults: { providers: { litellm: { baseUrl: 'http://localhost:4000/v1', models: [{ id: 'm' }] } } }, + skills: [{ name: 'a' }, { name: 'b', triggers: [{ type: 'local' }] }], + }); + const resolved = resolveSkillConfigs(config); + expect(resolved).toHaveLength(2); + for (const trigger of resolved) { + expect(trigger.providers?.['litellm']?.baseUrl).toBe('http://localhost:4000/v1'); + } +}); +``` + +(Reuse the existing `resolveSkillConfigs`/`WardenConfigSchema` imports already present in the test file.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm --filter @sentry/warden test -- config/loader.test.ts -t "propagates defaults.providers"` +Expected: FAIL (`trigger.providers` is undefined). + +- [ ] **Step 3: Implement** + +1. `config/loader.ts`: + - Add to `ResolvedTrigger` (after `runtime?` ~line 401): `providers?: ProvidersConfig;` + - In `resolveSkillConfigs`, after `const runtime = defaults?.runtime ?? 'pi';` (line 495): `const providers = defaults?.providers;` + - Add `providers,` to both pushed objects (next to `runtime,` at lines ~546 and ~581). + - Import `ProvidersConfig` from `./schema.js` (extend the existing schema import). + +2. Each runner entry point copies `providers` next to where it already copies `auxiliaryModel`/`runtime` into the options object it builds: + - `action/triggers/executor.ts:189-192` → add `providers: trigger.providers,` + - `action/workflow/schedule.ts:216-219` → add `providers: resolved.providers,` + - `action/workflow/pr-workflow.ts` → set `providers` on the runner options (near line 184) from `baseDefaults?.providers ?? repoDefaults?.providers`, and on `auxiliaryOptions` (line 313) + fix-eval options (lines 635/685) carry the same `providers`. + - `action/review/poster.ts` → the consolidate options (line 282) and dedup options (line 315) add `providers: ctx.providers` (add `providers?: ProvidersConfig` to the poster `ctx` type, populated upstream from the resolved trigger). + - `cli/main.ts` → add `providers: match?.providers ?? config?.defaults?.providers` to the runner options near line 1100, and `providers: t.providers` to the per-trigger options near line 1127; add `providers` to the local options interface (~615) and the `mergeOptions` allow-list (~646) if model fields are gated there. + +- [ ] **Step 4: Run tests + typecheck** + +Run: `pnpm --filter @sentry/warden test -- config/loader.test.ts -t "propagates defaults.providers"` → PASS. +Run: `pnpm --filter @sentry/warden typecheck` → no errors (catches any entry point still missing `providers` where the options type now requires it, or unused-field issues). + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/config/loader.ts packages/warden/src/action packages/warden/src/cli/main.ts packages/warden/src/config/loader.test.ts +git commit -m "feat(config): propagate custom providers to all runner entry points + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 7: Preflight auth validation for custom providers + +**Files:** +- Modify: `packages/warden/src/cli/main.ts:778-790` (the preflight auth path that already calls `verifyAuth`) +- Modify: `packages/warden/src/action/workflow/base.ts` (or the action preflight near the existing auth guard ~line 71) +- Test: `packages/warden/src/cli/main.test.ts` if present, else assert via a focused unit on `assertCustomProviderAuth` already covered in Task 2 — add one integration assertion in `cli/main.test.ts` only if the harness exists. + +**Interfaces:** +- Consumes: `assertCustomProviderAuth`, `buildPiProviderOptions` from `../sdk/runtimes/custom-provider.js`. +- Produces: a clear thrown error before analysis starts when a non-loopback custom provider has no key. + +- [ ] **Step 1: Write the failing test** + +If `cli/main.test.ts` exists with a preflight harness, add: + +```ts +it('fails preflight when a non-loopback custom provider has no key', async () => { + // Arrange runner options with runtime 'pi' and providers pointing at a remote URL, + // with the relevant env var unset. Expect the run to reject with /API key/. + await expect(runWithOptions({ + runtime: 'pi', + providers: { litellm: { baseUrl: 'https://gw.example.com/v1', models: [{ id: 'm' }] } }, + })).rejects.toThrow(/litellm/); +}); +``` + +If no such harness exists, skip the integration test (Task 2 already unit-tests `assertCustomProviderAuth`) and proceed to Step 3; record this in the commit body. + +- [ ] **Step 2: Run test to verify it fails (if written)** + +Run: `pnpm --filter @sentry/warden test -- cli/main.test.ts -t "fails preflight"` +Expected: FAIL (no preflight check yet). + +- [ ] **Step 3: Implement** + +In the CLI preflight (`cli/main.ts`, the function around line 778-790 that calls `verifyAuth`), after the existing auth handling, add (only when `runtime === 'pi'`): + +```ts +import { buildPiProviderOptions, assertCustomProviderAuth } from '../sdk/runtimes/custom-provider.js'; + +// ...inside the preflight, when options.runtime is 'pi': +assertCustomProviderAuth(buildPiProviderOptions(options.providers, process.env)); +``` + +Add the equivalent guard in the action preflight next to the existing API-key error in `action/workflow/base.ts` (~line 71), guarded by `runtime === 'pi'`. + +- [ ] **Step 4: Run tests** + +Run: `pnpm --filter @sentry/warden test -- cli` and `pnpm --filter @sentry/warden typecheck` → PASS / no errors. + +- [ ] **Step 5: Commit** + +```bash +git add packages/warden/src/cli/main.ts packages/warden/src/action/workflow/base.ts +git commit -m "feat(runtime): fail fast when a remote custom provider has no key + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Task 8: Documentation + +**Files:** +- Modify: `packages/docs/src/content/docs/config/models.mdx` (after the "Pi Model Selectors" section, before "Claude Runtime Models" ~line 58) + +**Interfaces:** none (docs only). + +- [ ] **Step 1: Add the docs section** + +Insert into `packages/docs/src/content/docs/config/models.mdx`: + +````mdx +## Self-hosted / OpenAI-compatible providers (LiteLLM) + +When `runtime = "pi"`, you can register a self-hosted, OpenAI-compatible endpoint +(such as a [LiteLLM](https://docs.litellm.ai/) proxy) as a named provider and +target its models with the usual `provider/model` selector. The custom provider +covers every model lane (agent, auxiliary, synthesis). + +```toml title="warden.toml" +[defaults] +runtime = "pi" + +[defaults.providers.litellm] +baseUrl = "http://localhost:4000/v1" # required; OpenAI-compatible base URL +api = "openai-completions" # optional; default +# headers = { "X-Tenant" = "team-a" } # optional custom headers +# apiKeyEnv = "WARDEN_LITELLM_API_KEY" # optional; overrides the default lookup + +[[defaults.providers.litellm.models]] +id = "my-model" # required; the model name your endpoint exposes +# contextWindow = 128000 # optional; default 128000 +# maxTokens = 8192 # optional; default 8192 +# reasoning = false # optional; default false +# cost = { input = 0, output = 0, cacheRead = 0, cacheWrite = 0 } + +[defaults.agent] +model = "litellm/my-model" +``` + +**Model defaults:** only `id` is required. Warden fills `name` (= `id`), +`reasoning` (`false`), `input` (`["text"]`), `contextWindow` (`128000`), +`maxTokens` (`8192`), and `cost` (all zeros). Self-hosted runs therefore report +`$0` cost unless you set explicit costs. + +**Authentication:** the API key is read from the environment, never from +`warden.toml`. Warden looks up `apiKeyEnv` if set, otherwise +`WARDEN__API_KEY` then `_API_KEY` (e.g. `WARDEN_LITELLM_API_KEY`). +A `localhost` / `127.0.0.1` base URL may run without a key; any other host +requires one or Warden fails before analysis starts. + +### Alternative: Pi `models.json` + +Advanced users can instead define a custom provider in Pi's own `models.json` +(the format Pi loads on startup). Warden's Pi runtime picks up any providers and +models defined there. Prefer the `warden.toml` block above unless you already +maintain a shared Pi configuration. +```` + +- [ ] **Step 2: Build the docs** + +Run: `pnpm --filter warden-docs build` (or the repo's docs build script) +Expected: build succeeds. + +- [ ] **Step 3: Commit** + +```bash +git add packages/docs/src/content/docs/config/models.mdx +git commit -m "docs: document self-hosted OpenAI-compatible providers (LiteLLM) + +Co-Authored-By: Claude Opus 4.8 " +``` + +--- + +## Final verification + +- [ ] Run `pnpm lint && pnpm build && pnpm test` from the repo root. All green. +- [ ] Manual smoke (optional, requires a reachable LiteLLM): set `WARDEN_LITELLM_API_KEY`, add the `[defaults.providers.litellm]` block with `model = "litellm/"`, run `warden run` on a small diff, confirm analysis completes against the custom endpoint. + +--- + +## Self-Review + +**Spec coverage:** +- Config surface (`[defaults.providers.*]`) → Task 1. ✓ +- Model defaults (only `id` required) → Task 2 (`buildPiProviderOptions`). ✓ +- Auth from env, no secrets in config, loopback exception, fail-fast preflight → Task 2 (`resolveProviderApiKey`/`assertCustomProviderAuth`) + Task 7. ✓ +- `getRuntimeProviderOptions('pi', …)` → Task 3. ✓ +- Pi adapter `registerProvider` at the `runPiPrompt` choke point → Task 4. ✓ +- All lanes (agent + auxiliary + synthesis) → Tasks 3 (request types) + 5 (call sites) + 6 (entry points). ✓ +- `models.json` passthrough documented → Task 8. ✓ +- Tests (schema, builder, provider options, registration, threading, propagation) → Tasks 1-6. ✓ +- Docs → Task 8. ✓ +- Out of scope (per-skill providers, ANTHROPIC_BASE_URL formalization, non-OpenAI compat) → not implemented, consistent with spec. ✓ + +**Placeholder scan:** No "TBD"/"implement later". The two places that say "if the harness exists" (Task 5 mock extension, Task 7 integration test) are conditional on existing test infrastructure and give a concrete fallback; not placeholders for missing plan content. + +**Type consistency:** `ProvidersConfig`/`ProviderConfig`/`ProviderModelConfig` (Task 1) are used verbatim in Tasks 2/3/5/6. `PiProviderOptions`/`PiProvider`/`PiProviderModel` (Task 2) are used verbatim in Tasks 3/4. `getRuntimeProviderOptions(name, { providers })` signature (Task 3) matches every call site in Tasks 5/7. `request.providerOptions` (Task 3 types) is read in Task 4 and written in Task 5. `registerProvider(name, { baseUrl, api, headers, apiKey, models })` matches the verified `ProviderConfigInput` shape. diff --git a/docs/superpowers/specs/2026-06-26-warden-custom-provider-litellm-design.md b/docs/superpowers/specs/2026-06-26-warden-custom-provider-litellm-design.md new file mode 100644 index 00000000..1777fa79 --- /dev/null +++ b/docs/superpowers/specs/2026-06-26-warden-custom-provider-litellm-design.md @@ -0,0 +1,219 @@ +# Design: Self-hosted LLMs via custom Pi providers (LiteLLM) + +**Date:** 2026-06-26 +**Status:** Approved (pending spec review) +**Topic:** Custom OpenAI-compatible provider support for the Pi runtime + +## Problem + +Warden can only target the providers Pi ships built-in (Anthropic, OpenAI, +OpenRouter, Fireworks, etc.) or the Claude runtime. Users who run their own +models behind a self-hosted gateway such as [LiteLLM](https://docs.litellm.ai/) +have no first-class way to point Warden at a custom base URL. LiteLLM exposes an +OpenAI-compatible `/v1/chat/completions` surface (and an Anthropic-compatible +`/v1/messages` surface), so the capability exists at the SDK layer but is not +reachable from `warden.toml`. + +## Goal + +Let a user register a self-hosted, OpenAI-compatible endpoint as a named provider +in `warden.toml` and select its models with the existing `provider/model` +selector syntax. All model lanes (agent, auxiliary, synthesis) route through the +custom provider when their selector points at it. + +## Non-goals (YAGNI) + +- Per-skill or per-trigger provider definitions. Providers are global within a + config layer, consistent with how `runtime` already works. +- Formalizing `ANTHROPIC_BASE_URL` for the `claude` runtime. (It already works + via env passthrough; not part of this change.) +- Compatibility modes beyond OpenAI-completions. The schema reserves an `api` + field so `anthropic-messages` can be added later, but only + `openai-completions` is wired and tested now. +- Storing secrets in `warden.toml`. API keys come from the environment only. + +## Key facts (verified against installed packages) + +- `@earendil-works/pi-coding-agent@0.78.0` `ModelRegistry` exposes + `registerProvider(name, config: ProviderConfigInput)` where + `ProviderConfigInput` accepts `{ baseUrl, apiKey, api, headers, authHeader, + models[] }`. Each `models[]` entry requires + `{ id, name, reasoning, input, cost{input,output,cacheRead,cacheWrite}, + contextWindow, maxTokens }` and optional `{ api, baseUrl, headers, compat, + thinkingLevelMap }`. +- `@earendil-works/pi-ai@0.78.0` `Api` includes `"openai-completions"`. + `OpenAICompletionsCompat` is auto-detected from the base URL when `compat` is + omitted. +- `pi.ts` `runPiPrompt()` is the single choke point: both `runSkill` and the + auxiliary/synthesis `runStructured` paths call it. It builds a fresh + `ModelRegistry.create(authStorage)` per call, then `resolvePiModel()`. + Registering the custom provider here covers every lane. +- `AuthStorage.setRuntimeApiKey(provider, apiKey)` already sets the legacy + Anthropic key today (`createAuthStorage`). The same mechanism sets a custom + provider key. +- `bridgeWardenProviderApiKeyEnv()` already mirrors `WARDEN__API_KEY` to + `_API_KEY`. +- The agent path (`analyze.ts`, `verify.ts`) already builds provider options via + `getRuntimeProviderOptions(runtimeName, {...})` and threads a `providerOptions` + field on `SkillRunRequest`. `AuxiliaryRunRequest`/`SynthesisRunRequest` do + **not** yet carry `providerOptions`; this change adds it. + +## Configuration surface + +New `[defaults.providers.]` map in `warden.toml`: + +```toml +[defaults] +runtime = "pi" + +[defaults.providers.litellm] +baseUrl = "http://localhost:4000/v1" # required; OpenAI-compatible base URL +api = "openai-completions" # optional; default "openai-completions" +# headers = { "X-Tenant" = "team-a" } # optional custom headers +# apiKeyEnv = "WARDEN_LITELLM_API_KEY" # optional; override the default env lookup + +[[defaults.providers.litellm.models]] +id = "my-model" # required; the model name LiteLLM exposes +# contextWindow = 128000 # optional; default 128000 +# maxTokens = 8192 # optional; default 8192 +# reasoning = false # optional; default false +# cost = { input = 0, output = 0, cacheRead = 0, cacheWrite = 0 } # default zeros + +[defaults.agent] +model = "litellm/my-model" +``` + +Selector behavior is unchanged: split at the first `/`, provider before, model id +after. `litellm/my-model` resolves to the registered `litellm` provider. + +### Model field defaults + +Only `id` is required per model. Warden fills the rest before calling +`registerProvider`: + +| Field | Default | +| --- | --- | +| `name` | same as `id` | +| `reasoning` | `false` | +| `input` | `["text"]` | +| `contextWindow` | `128000` | +| `maxTokens` | `8192` | +| `cost` | `{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }` | +| `api` | provider-level `api` (default `openai-completions`) | + +Cost defaults of zero mean self-hosted runs report `$0`; acceptable for the +common internal-gateway case. + +## Authentication + +No secret lives in `warden.toml`. The Bearer key is resolved from the +environment at runtime: + +1. If `apiKeyEnv` is set, read that variable. +2. Otherwise try `WARDEN__API_KEY`, then `_API_KEY` + (e.g. `WARDEN_LITELLM_API_KEY`, then `LITELLM_API_KEY`). +3. If none is set and the endpoint is non-loopback, fail preflight with a clear + message naming the expected env var. Loopback URLs (localhost/127.0.0.1) are + allowed to run unauthenticated. + +The resolved key is handed to Pi via `registerProvider({ apiKey })` and/or +`AuthStorage.setRuntimeApiKey(name, key)`. It never touches disk. + +## Architecture / plumbing + +### 1. Schema (`config/schema.ts`) + +- `ProviderModelSchema`: `{ id: string (min 1), name?: string, + reasoning?: boolean, input?: ("text"|"image")[], contextWindow?: positive int, + maxTokens?: positive int, cost?: { input, output, cacheRead, cacheWrite all + >= 0 } }`. +- `ProviderConfigSchema`: `{ baseUrl: z.string().url(), + api?: z.enum(["openai-completions"]) (default "openai-completions"), + headers?: z.record(z.string()), apiKeyEnv?: z.string(), + models: z.array(ProviderModelSchema).min(1) }` (`.strict()`). +- `ProvidersConfigSchema`: `z.record(z.string(), ProviderConfigSchema)`. +- Add `providers: ProvidersConfigSchema.optional()` to `DefaultsSchema`. + +### 2. Provider options builder (`runtimes/index.ts`) + +- Extend `RuntimeProviderOptionsInput` with an optional `providers` field + (the validated `ProvidersConfig`). +- In `getRuntimeProviderOptions`, when `name === 'pi'` and `providers` is + present, return a `PiProviderOptions` object: the normalized provider list with + model defaults applied and the env-resolved API key attached per provider. + Key resolution (env lookup) happens here so the Pi adapter receives ready + values; absent keys are left undefined for the adapter/preflight to handle. +- `claude` branch unchanged. + +### 3. Pi adapter (`runtimes/pi.ts`) + +- Add `providers?: PiProviderOptions` to `PiPromptOptions`. +- In `runPiPrompt`, after `ModelRegistry.create(authStorage)` and before + `resolvePiModel`, iterate `providers` and call + `modelRegistry.registerProvider(name, { baseUrl, api, headers, apiKey, + models })`. Set the runtime API key on `authStorage` when present. +- `runSkill` reads `providers` from `request.providerOptions`. +- `runStructured` (auxiliary/synthesis) reads `providers` from the request and + forwards into `runPiPrompt`. + +### 4. Request types (`runtimes/types.ts`) + +- Add `providerOptions?: unknown` to `AuxiliaryRunRequestBase` and + `SynthesisRunRequest` so auxiliary/synthesis calls can carry the same + provider options the agent path already passes. + +### 5. Threading (all lanes) + +Pass the `providers` config into `getRuntimeProviderOptions` at every Pi call +site, mirroring how `runtimeName`/`model` already flow: + +- Agent: `analyze.ts`, `verify.ts` (extend the existing + `getRuntimeProviderOptions` calls with `providers`). +- Auxiliary/synthesis: `extract.ts`, `output/dedup.ts`, + `action/fix-evaluation/judge.ts`, `sdk/json-output.ts`, and the skill-builder + paths (`outline.ts`, `agentic.ts`, `skill.ts`) — add `providerOptions: + getRuntimeProviderOptions(runtimeName, { providers })` to each runtime request. + +The `providers` value originates from the resolved `WardenConfig.defaults` and +is carried alongside the existing runner options object that already conveys +`runtime`/`model` to these call sites. + +### 6. Validation & errors + +- When resolving a model selector whose provider prefix is neither a built-in Pi + provider nor a key in `[defaults.providers]`, throw a clear error naming the + unknown provider and listing configured custom providers. Reuse the existing + invalid-selector error surface where practical. +- Preflight: if a configured provider requires a key (has `apiKeyEnv` or a + non-loopback `baseUrl`) and none resolves, fail before the first model call. + +## Testing + +- **Unit (`config/schema.test.ts`)**: accept a valid `[defaults.providers.*]` + block; reject a bad `baseUrl`, an unsupported `api`, and an empty `models` + array. +- **Unit (`runtimes/index.test.ts`)**: `getRuntimeProviderOptions('pi', + { providers })` applies model defaults and resolves the API key from + `apiKeyEnv`, then `WARDEN__API_KEY`, then `_API_KEY`; returns + undefined key when absent. +- **Integration (`runtimes/pi.test.ts`)**: with `ModelRegistry`/ + `registerProvider` stubbed, a request carrying a custom provider registers it + and resolves `litellm/my-model`. Mock the HTTP boundary; no live LiteLLM. +- **Regression**: a selector with an unknown provider prefix produces the + friendly error. + +## Documentation + +Extend `packages/docs/src/content/docs/config/models.mdx` with a "Self-hosted / +OpenAI-compatible providers (LiteLLM)" section covering the +`[defaults.providers.*]` block, the model-field defaults table, env-var auth, +and the Pi `models.json` passthrough as an alternative escape hatch. + +## Decisions captured + +- Runtime: Pi only. +- Config surface: first-class `warden.toml` + documented `models.json` + passthrough. +- Endpoint shape: OpenAI-compatible with Bearer key. +- Lane coverage: all lanes (agent + auxiliary + synthesis). +- Model fields: sensible defaults; only `id` required. diff --git a/packages/docs/src/content/docs/config/models.mdx b/packages/docs/src/content/docs/config/models.mdx index 4a79f08e..c84a0f40 100644 --- a/packages/docs/src/content/docs/config/models.mdx +++ b/packages/docs/src/content/docs/config/models.mdx @@ -56,6 +56,52 @@ Warden mirrors these to the native `{PROVIDER}_API_KEY` expected by each SDK at If you already have a native provider key set (e.g. `OPENAI_API_KEY`), Warden will use it directly and the `WARDEN_`-prefixed form is not required. +## Self-hosted / OpenAI-compatible providers (LiteLLM) + +When `runtime = "pi"`, you can register a self-hosted, OpenAI-compatible endpoint +(such as a [LiteLLM](https://docs.litellm.ai/) proxy) as a named provider and +target its models with the usual `provider/model` selector. The custom provider +covers every model lane (agent, auxiliary, synthesis). + +```toml title="warden.toml" +[defaults] +runtime = "pi" + +[defaults.providers.litellm] +baseUrl = "http://localhost:4000/v1" # required; OpenAI-compatible base URL +api = "openai-completions" # optional; default +# headers = { "X-Tenant" = "team-a" } # optional custom headers +# apiKeyEnv = "WARDEN_LITELLM_API_KEY" # optional; overrides the default lookup + +[[defaults.providers.litellm.models]] +id = "my-model" # required; the model name your endpoint exposes +# contextWindow = 128000 # optional; default 128000 +# maxTokens = 8192 # optional; default 8192 +# reasoning = false # optional; default false +# cost = { input = 0, output = 0, cacheRead = 0, cacheWrite = 0 } + +[defaults.agent] +model = "litellm/my-model" +``` + +**Model defaults:** only `id` is required. Warden fills `name` (= `id`), +`reasoning` (`false`), `input` (`["text"]`), `contextWindow` (`128000`), +`maxTokens` (`8192`), and `cost` (all zeros). Self-hosted runs therefore report +`$0` cost unless you set explicit costs. + +**Authentication:** the API key is read from the environment, never from +`warden.toml`. Warden looks up `apiKeyEnv` if set, otherwise +`WARDEN__API_KEY` then `_API_KEY` (e.g. `WARDEN_LITELLM_API_KEY`). +A loopback base URL (`localhost`, `127.0.0.1`, or `::1`) may run without a key; +any other host requires one or Warden fails before analysis starts. + +### Alternative: Pi `models.json` + +Advanced users can instead define a custom provider in Pi's own `models.json` +(the format Pi loads on startup). Warden's Pi runtime picks up any providers and +models defined there. Prefer the `warden.toml` block above unless you already +maintain a shared Pi configuration. + ## Claude Runtime Models When `runtime = "claude"`, use the model IDs accepted by Claude Code: @@ -156,8 +202,19 @@ Main agent model precedence, from highest to lowest: | `WARDEN_MODEL` | Environment fallback. | | SDK/runtime default | Used when no explicit model is set. | -Auxiliary and synthesis models only come from `[defaults.auxiliary]` and -`[defaults.synthesis]`. They do not inherit skill or trigger `model` overrides. +The auxiliary lane (structured extraction, dedup, merge, fix evaluation) and the +synthesis lane prefer `[defaults.auxiliary]` and `[defaults.synthesis]`. When +unset, they fall back to the global default model +(`defaults.agent.model` → `defaults.model` → `--model` → `WARDEN_MODEL`), so a +single configured model drives every lane. Synthesis falls back to the auxiliary +model before the global default. These lanes do not inherit skill- or +trigger-level `model` overrides, which apply to the agent lane only. + +This matters for self-hosted providers: setting just `defaults.model` to a +custom-provider model (e.g. `litellm/...`) keeps the auxiliary and synthesis +lanes on that provider too, rather than escaping to a runtime default on another +provider. Set `[defaults.auxiliary]` explicitly only when you want those lanes on +a different (e.g. cheaper) model. Effort is separate from model selection. For local runs, `--effort` overrides `defaults.agent.effort` for that diff --git a/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts b/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts index aef185ac..40913140 100644 --- a/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts +++ b/packages/warden/src/action/fix-evaluation/judge.runtime-options.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { Octokit } from '@octokit/rest'; import type { ExistingComment } from '../../output/dedup.js'; +import type { ProvidersConfig } from '../../config/schema.js'; import type { FixJudgeContext, FixJudgeInput, FixJudgeRuntimeOptions } from './judge.js'; describe('evaluateFix runtime options', () => { @@ -19,6 +20,7 @@ describe('evaluateFix runtime options', () => { vi.doMock('../../sdk/runtimes/index.js', () => ({ getRuntime, + getRuntimeProviderOptions: vi.fn(() => undefined), })); const { evaluateFix } = await import('./judge.js'); @@ -57,7 +59,9 @@ describe('evaluateFix runtime options', () => { ); expect(result.usedFallback).toBe(false); - expect(getRuntime).toHaveBeenCalledWith(undefined); + // Omitted runtime resolves to 'pi' (getRuntime's default), and the same + // value drives the provider-options lookup so the two never diverge. + expect(getRuntime).toHaveBeenCalledWith('pi'); expect(runAuxiliary).toHaveBeenCalledWith( expect.objectContaining({ model: undefined, @@ -68,7 +72,130 @@ describe('evaluateFix runtime options', () => { expect(runAuxiliary).toHaveBeenCalledWith( expect.objectContaining({ prompt: expect.stringContaining(''), + providerOptions: undefined, }) ); }); + + it('forwards resolved provider options to runAuxiliary', async () => { + const runAuxiliary = vi.fn().mockResolvedValue({ + success: true, + data: { status: 'not_attempted', reasoning: 'No related changes' }, + usage: { inputTokens: 0, outputTokens: 0, costUSD: 0 }, + }); + const getRuntime = vi.fn(() => ({ runAuxiliary })); + const providerOptions = { providers: [{ name: 'litellm' }] }; + const getRuntimeProviderOptions = vi.fn(() => providerOptions); + + vi.doMock('../../sdk/runtimes/index.js', () => ({ + getRuntime, + getRuntimeProviderOptions, + })); + + const { evaluateFix } = await import('./judge.js'); + + const comment: ExistingComment = { + id: 1, + path: 'src/handler.ts', + line: 12, + title: 'SQL injection', + description: 'User input is concatenated into SQL', + contentHash: 'abc123', + isWarden: true, + threadId: 'thread-1', + }; + + const input: FixJudgeInput = { + comment, + changedFiles: ['src/handler.ts'], + codeBeforeFix: '12: const query = "SELECT * FROM users WHERE id = " + id;', + }; + + const context: FixJudgeContext = { + octokit: {} as Octokit, + owner: 'test-owner', + repo: 'test-repo', + baseSha: 'base123', + headSha: 'head456', + patches: new Map(), + }; + + const providers: ProvidersConfig = { + litellm: { + baseUrl: 'http://localhost:4000', + api: 'openai-completions', + models: [{ id: 'litellm/gpt-4o', name: 'gpt-4o' }], + }, + }; + + await evaluateFix(input, context, 'api-key', { runtime: 'pi', providers }); + + expect(getRuntime).toHaveBeenCalledWith('pi'); + expect(getRuntimeProviderOptions).toHaveBeenCalledWith('pi', { providers }); + expect(runAuxiliary).toHaveBeenCalledWith( + expect.objectContaining({ providerOptions }) + ); + }); + + it('builds provider options for the resolved runtime when runtime is omitted', async () => { + const runAuxiliary = vi.fn().mockResolvedValue({ + success: true, + data: { status: 'not_attempted', reasoning: 'No related changes' }, + usage: { inputTokens: 0, outputTokens: 0, costUSD: 0 }, + }); + const getRuntime = vi.fn(() => ({ runAuxiliary })); + const providerOptions = { providers: [{ name: 'litellm' }] }; + const getRuntimeProviderOptions = vi.fn(() => providerOptions); + + vi.doMock('../../sdk/runtimes/index.js', () => ({ + getRuntime, + getRuntimeProviderOptions, + })); + + const { evaluateFix } = await import('./judge.js'); + + const comment: ExistingComment = { + id: 1, + path: 'src/handler.ts', + line: 12, + title: 'SQL injection', + description: 'User input is concatenated into SQL', + contentHash: 'abc123', + isWarden: true, + threadId: 'thread-1', + }; + + const input: FixJudgeInput = { + comment, + changedFiles: ['src/handler.ts'], + codeBeforeFix: '12: const query = "SELECT * FROM users WHERE id = " + id;', + }; + + const context: FixJudgeContext = { + octokit: {} as Octokit, + owner: 'test-owner', + repo: 'test-repo', + baseSha: 'base123', + headSha: 'head456', + patches: new Map(), + }; + + const providers: ProvidersConfig = { + litellm: { + baseUrl: 'http://localhost:4000', + api: 'openai-completions', + models: [{ id: 'litellm/gpt-4o', name: 'gpt-4o' }], + }, + }; + + // No runtime specified: it resolves to 'pi', and the provider-options lookup + // must use that same 'pi' so custom providers are built, not dropped. + await evaluateFix(input, context, 'api-key', { providers }); + + expect(getRuntime).toHaveBeenCalledWith('pi'); + expect(getRuntimeProviderOptions).toHaveBeenCalledWith('pi', { providers }); + expect(runAuxiliary).toHaveBeenCalledWith( + expect.objectContaining({ providerOptions }) + ); + }); }); diff --git a/packages/warden/src/action/fix-evaluation/judge.ts b/packages/warden/src/action/fix-evaluation/judge.ts index 1b069d7f..e5970c62 100644 --- a/packages/warden/src/action/fix-evaluation/judge.ts +++ b/packages/warden/src/action/fix-evaluation/judge.ts @@ -7,7 +7,8 @@ import { buildTaggedSection, joinPromptSections, } from '../../sdk/prompt-sections.js'; -import { getRuntime, type AuxiliaryTool, type RuntimeName } from '../../sdk/runtimes/index.js'; +import { getRuntime, getRuntimeProviderOptions, type AuxiliaryTool, type RuntimeName } from '../../sdk/runtimes/index.js'; +import type { ProvidersConfig } from '../../config/schema.js'; import { emptyUsage } from '../../sdk/usage.js'; import { FixJudgeVerdictSchema } from './types.js'; import type { FixJudgeResult } from './types.js'; @@ -33,6 +34,8 @@ export interface FixJudgeContext { export interface FixJudgeRuntimeOptions { runtime?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; model?: string; maxRetries?: number; } @@ -223,7 +226,13 @@ export async function evaluateFix( const prompt = buildPrompt(input); const executeTool = createToolExecutor(context); - const result = await getRuntime(runtimeOptions.runtime).runAuxiliary({ + // Resolve one effective runtime so getRuntime and getRuntimeProviderOptions + // never diverge. getRuntime defaults an omitted runtime to 'pi'; previously the + // provider-options lookup defaulted to 'claude', so on the omitted-runtime path + // the call ran on Pi but built Claude-shaped options and silently dropped any + // custom providers. + const runtime = runtimeOptions.runtime ?? 'pi'; + const result = await getRuntime(runtime).runAuxiliary({ task: 'fix_evaluation', agentName: input.skillName, apiKey, @@ -234,6 +243,7 @@ export async function evaluateFix( model: runtimeOptions.model, maxIterations: 5, maxRetries: runtimeOptions.maxRetries, + providerOptions: getRuntimeProviderOptions(runtime, { providers: runtimeOptions.providers }), }); if (result.success) { diff --git a/packages/warden/src/action/review/poster.ts b/packages/warden/src/action/review/poster.ts index 29d00774..e2a1e048 100644 --- a/packages/warden/src/action/review/poster.ts +++ b/packages/warden/src/action/review/poster.ts @@ -21,6 +21,7 @@ import type { ExistingComment, DeduplicateResult } from '../../output/dedup.js'; import { mergeAuxiliaryUsage, mergeAuxiliaryUsageAttribution } from '../../sdk/usage.js'; import { canUseRuntimeAuth } from '../../sdk/extract.js'; import type { RuntimeName } from '../../sdk/runtimes/index.js'; +import type { ProvidersConfig } from '../../config/schema.js'; import type { TriggerResult } from '../triggers/executor.js'; import { logAction, warnAction } from '../../cli/output/tty.js'; import type { FindingObservation } from '../reporting/outcomes.js'; @@ -37,6 +38,7 @@ export interface ReviewPostingContext { existingComments: ExistingComment[]; apiKey: string; runtime?: RuntimeName; + providers?: ProvidersConfig; model?: string; maxRetries?: number; /** Throw review posting failures instead of converting them to warnings. */ @@ -258,6 +260,7 @@ export async function postTriggerReview( const consolidateResult = await consolidateBatchFindings(findingsToPost, { apiKey, runtime: ctx.runtime, + providers: ctx.providers, model: ctx.model, hashOnly: !canUseAuxiliaryRuntime, maxRetries: ctx.maxRetries, @@ -297,6 +300,7 @@ export async function postTriggerReview( dedupResult = await deduplicateFindings(findingsToPost, existingComments, { apiKey, runtime: ctx.runtime, + providers: ctx.providers, model: ctx.model, currentSkill: skill, maxRetries: ctx.maxRetries, diff --git a/packages/warden/src/action/triggers/executor.ts b/packages/warden/src/action/triggers/executor.ts index 90affb87..846a1818 100644 --- a/packages/warden/src/action/triggers/executor.ts +++ b/packages/warden/src/action/triggers/executor.ts @@ -24,6 +24,10 @@ import type { Semaphore } from '../../utils/index.js'; import { Verbosity } from '../../cli/output/verbosity.js'; import type { ProviderFailureCircuitBreaker } from '../../sdk/circuit-breaker.js'; import { assertValidPiModelSelectors } from '../../sdk/runtimes/model-selectors.js'; +import { + buildPiProviderOptions, + assertCustomProviderAuth, +} from '../../sdk/runtimes/custom-provider.js'; import { captureActionTriggerError } from '../error-reporting.js'; /** Log-mode output for CI: no TTY, no color. */ @@ -174,6 +178,10 @@ export async function executeTrigger( try { assertValidPiModelSelectors([trigger]); + if ((trigger.runtime ?? 'pi') === 'pi') { + assertCustomProviderAuth(buildPiProviderOptions(trigger.providers, process.env)); + } + const taskOptions: SkillTaskOptions = { name: trigger.name, displayName: trigger.skill, @@ -187,6 +195,7 @@ export async function executeTrigger( apiKey: anthropicApiKey, model: trigger.model, runtime: trigger.runtime, + providers: trigger.providers, effort: trigger.effort, auxiliaryModel: trigger.auxiliaryModel, synthesisModel: trigger.synthesisModel, diff --git a/packages/warden/src/action/workflow/pr-workflow.test.ts b/packages/warden/src/action/workflow/pr-workflow.test.ts index bb7d2cf0..dc8f7df5 100644 --- a/packages/warden/src/action/workflow/pr-workflow.test.ts +++ b/packages/warden/src/action/workflow/pr-workflow.test.ts @@ -119,7 +119,9 @@ import { runSkillTask } from '../../cli/output/tasks.js'; import { fetchExistingComments, deduplicateFindings, processDuplicateActions } from '../../output/dedup.js'; import { evaluateFixAttempts } from '../fix-evaluation/index.js'; import { setFailed, writeFindingsOutput } from './base.js'; -import { runPRWorkflow } from './pr-workflow.js'; +import { runPRWorkflow, resolveWorkflowAuxiliaryOptions } from './pr-workflow.js'; +import type { WardenConfig } from '../../config/schema.js'; +import type { LoadedLayeredConfig } from '../../config/loader.js'; import { clearSkillsCache } from '../../skills/loader.js'; import { Semaphore } from '../../utils/index.js'; import { buildFindingsOutput } from '../reporting/output.js'; @@ -2264,4 +2266,45 @@ describe('runPRWorkflow', () => { expect(mockEvaluateFixAttempts).not.toHaveBeenCalled(); }); }); + + describe('resolveWorkflowAuxiliaryOptions', () => { + const layered = (config: WardenConfig, baseConfig?: WardenConfig, repoConfig?: WardenConfig): LoadedLayeredConfig => + ({ config, baseConfig, repoConfig } as LoadedLayeredConfig); + + it('inherits the global default model when no auxiliary model is set', () => { + const options = resolveWorkflowAuxiliaryOptions( + layered({ version: 1, skills: [], defaults: { model: 'litellm/gemma' } }), + ); + expect(options.model).toBe('litellm/gemma'); + }); + + it('inherits the agent model when no auxiliary model is set', () => { + const options = resolveWorkflowAuxiliaryOptions( + layered({ version: 1, skills: [], defaults: { agent: { model: 'litellm/gemma' } } }), + ); + expect(options.model).toBe('litellm/gemma'); + }); + + it('prefers an explicit auxiliary model over the inherited global model', () => { + const options = resolveWorkflowAuxiliaryOptions( + layered({ + version: 1, + skills: [], + defaults: { model: 'litellm/gemma', auxiliary: { model: 'litellm/cheap' } }, + }), + ); + expect(options.model).toBe('litellm/cheap'); + }); + + it('inherits the base layer global model when the repo layer omits it', () => { + const options = resolveWorkflowAuxiliaryOptions( + layered( + { version: 1, skills: [] }, + { version: 1, skills: [], defaults: { model: 'litellm/org' } }, + { version: 1, skills: [], defaults: { runtime: 'pi' } }, + ), + ); + expect(options.model).toBe('litellm/org'); + }); + }); }); diff --git a/packages/warden/src/action/workflow/pr-workflow.ts b/packages/warden/src/action/workflow/pr-workflow.ts index 7546b383..18533664 100644 --- a/packages/warden/src/action/workflow/pr-workflow.ts +++ b/packages/warden/src/action/workflow/pr-workflow.ts @@ -45,6 +45,7 @@ import { postTriggerReview } from '../review/poster.js'; import { shouldResolveStaleComments } from '../review/coordination.js'; import type { FindingObservation } from '../reporting/outcomes.js'; import type { RuntimeName } from '../../sdk/runtimes/index.js'; +import type { ProvidersConfig } from '../../config/schema.js'; import { canUseRuntimeAuth } from '../../sdk/extract.js'; import { ProviderFailureCircuitBreaker } from '../../sdk/circuit-breaker.js'; import { @@ -119,6 +120,7 @@ interface FixEvaluationCommentGroups { interface AuxiliaryWorkflowOptions { runtime?: RuntimeName; + providers?: ProvidersConfig; model?: string; maxRetries?: number; } @@ -173,7 +175,7 @@ function checkOptionsForPullRequest(context: EventContext): CheckOptions | undef }; } -function resolveWorkflowAuxiliaryOptions(layered: LoadedLayeredConfig): AuxiliaryWorkflowOptions { +export function resolveWorkflowAuxiliaryOptions(layered: LoadedLayeredConfig): AuxiliaryWorkflowOptions { const baseDefaults = layered.baseConfig?.defaults; const repoDefaults = layered.repoConfig?.defaults ?? layered.config.defaults; @@ -182,9 +184,18 @@ function resolveWorkflowAuxiliaryOptions(layered: LoadedLayeredConfig): Auxiliar // trigger, so the org base config remains the enforced baseline and the // repo layer only fills fields the base omits. runtime: baseDefaults?.runtime ?? repoDefaults?.runtime ?? 'pi', + providers: baseDefaults?.providers ?? repoDefaults?.providers, + // Inherit the global default model when no explicit auxiliary model is set, + // so workflow-scoped helper calls stay on the configured provider instead of + // falling back to a runtime default on another provider. Base-first to match + // the enforced-baseline precedence above; explicit auxiliary models win. model: emptyToUndefined(baseDefaults?.auxiliary?.model) ?? - emptyToUndefined(repoDefaults?.auxiliary?.model), + emptyToUndefined(repoDefaults?.auxiliary?.model) ?? + emptyToUndefined(baseDefaults?.agent?.model) ?? + emptyToUndefined(baseDefaults?.model) ?? + emptyToUndefined(repoDefaults?.agent?.model) ?? + emptyToUndefined(repoDefaults?.model), maxRetries: baseDefaults?.auxiliary?.maxRetries ?? baseDefaults?.auxiliaryMaxRetries ?? @@ -564,6 +575,7 @@ async function postReviewsAndTrackFailures( existingComments, apiKey: inputs.anthropicApiKey, runtime: auxiliaryOptions.runtime, + providers: auxiliaryOptions.providers, model: auxiliaryOptions.model, maxRetries: auxiliaryOptions.maxRetries, failOnPostError: options.failOnPostError, diff --git a/packages/warden/src/action/workflow/schedule.ts b/packages/warden/src/action/workflow/schedule.ts index ba3d36ea..eb251e9b 100644 --- a/packages/warden/src/action/workflow/schedule.ts +++ b/packages/warden/src/action/workflow/schedule.ts @@ -16,6 +16,10 @@ import type { ScheduleConfig } from '../../config/schema.js'; import { buildScheduleEventContext } from '../../event/schedule-context.js'; import { runSkill } from '../../sdk/runner.js'; import { assertValidPiModelSelectors } from '../../sdk/runtimes/model-selectors.js'; +import { + buildPiProviderOptions, + assertCustomProviderAuth, +} from '../../sdk/runtimes/custom-provider.js'; import { createOrUpdateIssue } from '../../output/github-issues.js'; import { shouldFail, countFindingsAtOrAbove, countSeverity } from '../../triggers/matcher.js'; import { resolveSkillAsync } from '../../skills/loader.js'; @@ -179,6 +183,10 @@ async function runScheduleWorkflowInner( try { assertValidPiModelSelectors([resolved]); + if ((resolved.runtime ?? 'pi') === 'pi') { + assertCustomProviderAuth(buildPiProviderOptions(resolved.providers, process.env)); + } + // Build context from paths filter const patterns = resolved.filters?.paths ?? ['**/*']; const ignorePatterns = resolved.filters?.ignorePaths; @@ -214,6 +222,7 @@ async function runScheduleWorkflowInner( apiKey: inputs.anthropicApiKey, model: resolved.model, runtime: resolved.runtime, + providers: resolved.providers, effort: resolved.effort, auxiliaryModel: resolved.auxiliaryModel, synthesisModel: resolved.synthesisModel, diff --git a/packages/warden/src/cli/commands/build.test.ts b/packages/warden/src/cli/commands/build.test.ts index eb086428..6188d8e6 100644 --- a/packages/warden/src/cli/commands/build.test.ts +++ b/packages/warden/src/cli/commands/build.test.ts @@ -5,7 +5,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { CLIOptions } from '../args.js'; import { Reporter } from '../output/reporter.js'; import { Verbosity } from '../output/verbosity.js'; -import { runBuild, runImprove } from './build.js'; +import { runBuild, runImprove, resolveSynthesisModel, resolveRepairModel } from './build.js'; +import type { WardenConfig } from '../../config/schema.js'; import { getRepoRoot } from '../git.js'; import { buildGeneratedSkillDefinition, @@ -490,4 +491,59 @@ prompt: |- expect(buildSkillOutlineMock).not.toHaveBeenCalled(); expect(buildGeneratedSkillMock).not.toHaveBeenCalled(); }); + + it('fails fast when a remote custom provider has no key', async () => { + const reporter = createTestReporter(); + vi.spyOn(console, 'error').mockImplementation(() => undefined); + // Hermetic: ensure no ambient key resolves for this provider name. + delete process.env['WARDEN_SMOKETESTPROVIDER_API_KEY']; + delete process.env['SMOKETESTPROVIDER_API_KEY']; + writeFileSync(join(tempDir, 'warden.toml'), [ + 'version = 1', + '[defaults]', + 'runtime = "pi"', + 'model = "smoketestprovider/model"', + '[defaults.providers.smoketestprovider]', + 'baseUrl = "http://remote.example.com/v1"', + 'api = "openai-completions"', + '[[defaults.providers.smoketestprovider.models]]', + 'id = "model"', + ].join('\n'), 'utf-8'); + + const exitCode = await runBuild(createOptions(), reporter); + + expect(exitCode).toBe(1); + // Preflight fails before any synthesis work touches the provider. + expect(buildSkillOutlineMock).not.toHaveBeenCalled(); + expect(buildGeneratedSkillMock).not.toHaveBeenCalled(); + }); +}); + +describe('build model resolution', () => { + const cfg = (defaults: NonNullable): WardenConfig => + ({ version: 1, skills: [], defaults }); + + it('synthesis lane inherits the global default model when synthesis and auxiliary are unset', () => { + expect(resolveSynthesisModel(cfg({ model: 'litellm/gemma' }), createOptions())).toBe('litellm/gemma'); + }); + + it('synthesis lane inherits the agent model when unset', () => { + expect(resolveSynthesisModel(cfg({ agent: { model: 'litellm/gemma' } }), createOptions())).toBe('litellm/gemma'); + }); + + it('synthesis lane prefers an explicit synthesis model over the inherited model', () => { + expect( + resolveSynthesisModel(cfg({ model: 'litellm/gemma', synthesis: { model: 'litellm/synth' } }), createOptions()), + ).toBe('litellm/synth'); + }); + + it('repair lane inherits the global default model when auxiliary is unset', () => { + expect(resolveRepairModel(cfg({ model: 'litellm/gemma' }), createOptions())).toBe('litellm/gemma'); + }); + + it('repair lane prefers an explicit auxiliary model over the inherited model', () => { + expect( + resolveRepairModel(cfg({ model: 'litellm/gemma', auxiliary: { model: 'litellm/aux' } }), createOptions()), + ).toBe('litellm/aux'); + }); }); diff --git a/packages/warden/src/cli/commands/build.ts b/packages/warden/src/cli/commands/build.ts index 48a0f38b..d61f66a5 100644 --- a/packages/warden/src/cli/commands/build.ts +++ b/packages/warden/src/cli/commands/build.ts @@ -36,6 +36,10 @@ import { invalidPiModelSelectorMessage, type InvalidPiModelSelector, } from '../../sdk/runtimes/model-selectors.js'; +import { + assertCustomProviderAuth, + buildPiProviderOptions, +} from '../../sdk/runtimes/custom-provider.js'; function renderHeader(args: { reporter: Reporter; @@ -157,18 +161,45 @@ function resolvePromptValue(prompt: string): string { return prompt.trim(); } -function resolveSynthesisModel( +/** + * Global default model chain shared by the build/improve lanes, mirroring the + * agent-lane resolution used elsewhere (agent.model -> model -> --model -> + * WARDEN_MODEL). Keeps the synthesis and repair lanes on the configured provider + * instead of escaping to a runtime default on another provider. + */ +function resolveDefaultModel( config: WardenConfig | undefined, options: CLIOptions, ): string | undefined { return ( - emptyToUndefined(config?.defaults?.synthesis?.model) ?? - emptyToUndefined(config?.defaults?.auxiliary?.model) ?? + emptyToUndefined(config?.defaults?.agent?.model) ?? + emptyToUndefined(config?.defaults?.model) ?? emptyToUndefined(options.model) ?? emptyToUndefined(process.env['WARDEN_MODEL']) ); } +export function resolveSynthesisModel( + config: WardenConfig | undefined, + options: CLIOptions, +): string | undefined { + return ( + emptyToUndefined(config?.defaults?.synthesis?.model) ?? + emptyToUndefined(config?.defaults?.auxiliary?.model) ?? + resolveDefaultModel(config, options) + ); +} + +export function resolveRepairModel( + config: WardenConfig | undefined, + options: CLIOptions, +): string | undefined { + return ( + emptyToUndefined(config?.defaults?.auxiliary?.model) ?? + resolveDefaultModel(config, options) + ); +} + type GeneratedSkillCommandMode = 'build' | 'improve'; function reportInvalidPiModelSelector(reporter: Reporter, invalid: InvalidPiModelSelector): void { @@ -331,8 +362,9 @@ async function runGeneratedSkillCommand( : collectSkillBuildSource(skill); const runtimeName = config?.defaults?.runtime ?? 'pi'; + const providers = config?.defaults?.providers; const model = resolveSynthesisModel(config, options); - const repairModel = emptyToUndefined(config?.defaults?.auxiliary?.model); + const repairModel = resolveRepairModel(config, options); const maxRetries = config?.defaults?.auxiliary?.maxRetries ?? config?.defaults?.auxiliaryMaxRetries; const invalidModelSelector = findInvalidPiModelSelector([{ runtime: runtimeName, @@ -343,6 +375,17 @@ async function runGeneratedSkillCommand( reportInvalidPiModelSelector(reporter, invalidModelSelector); return 1; } + // Fail fast when a remote custom provider has no resolvable key, matching the + // CLI, executor, and workflow entry points. Otherwise build would call the + // provider and fail mid-synthesis instead. + if (runtimeName === 'pi') { + try { + assertCustomProviderAuth(buildPiProviderOptions(providers, process.env)); + } catch (error) { + reporter.error(error instanceof Error ? error.message : String(error)); + return 1; + } + } const runtime = getRuntime(runtimeName); if (!options.json) { @@ -387,6 +430,7 @@ async function runGeneratedSkillCommand( source, repairModel, repairMaxRetries: maxRetries, + providers, onStatus: setDetail, }), }); @@ -435,6 +479,7 @@ async function runGeneratedSkillCommand( repairMaxRetries: maxRetries, abortController: state?.abortController, regenerate: options.regenerate || outlineResult.source === 'generated' || mode === 'improve', + providers, onStatus: setDetail, }), }); diff --git a/packages/warden/src/cli/main.test.ts b/packages/warden/src/cli/main.test.ts index 0b7322c7..ed923a7d 100644 --- a/packages/warden/src/cli/main.test.ts +++ b/packages/warden/src/cli/main.test.ts @@ -18,6 +18,7 @@ import { appendReportToRunLog, buildFinalChunkRecords, renderFinalRunLogContent, + verifyCustomProviderAuthForRun, type RunLog, type RunSkillSpec, } from './main.js'; @@ -666,6 +667,38 @@ describe('resolveCliDefaultModel', () => { expect(model).toBeUndefined(); }); + it('inherits the top-level model for the auxiliary lane when unset', () => { + expect( + resolveCliDefaultAuxiliaryModel({ defaults: { model: 'litellm/gemma' } }), + ).toBe('litellm/gemma'); + }); + + it('inherits the agent model for the auxiliary lane when unset', () => { + expect( + resolveCliDefaultAuxiliaryModel({ defaults: { agent: { model: 'litellm/gemma' } } }), + ).toBe('litellm/gemma'); + }); + + it('inherits the --model flag for the auxiliary lane when nothing else is set', () => { + expect( + resolveCliDefaultAuxiliaryModel({ defaults: {} }, 'litellm/gemma'), + ).toBe('litellm/gemma'); + }); + + it('prefers an explicit auxiliary model over the inherited top-level model', () => { + expect( + resolveCliDefaultAuxiliaryModel({ + defaults: { model: 'litellm/gemma', auxiliary: { model: 'litellm/cheap' } }, + }), + ).toBe('litellm/cheap'); + }); + + it('inherits the top-level model for synthesis through the auxiliary fallback', () => { + expect( + resolveCliDefaultSynthesisModel({ defaults: { model: 'litellm/gemma' } }), + ).toBe('litellm/gemma'); + }); + it('prefers synthesis defaults and falls back to auxiliary defaults', () => { const explicit = resolveCliDefaultSynthesisModel({ defaults: { @@ -828,3 +861,51 @@ describe('resolveInvocationCwd', () => { expect(resolveInvocationCwd('/launcher', '../repo')).toBe('/repo'); }); }); + +describe('verifyCustomProviderAuthForRun', () => { + const remoteProviders = { + litellm: { + baseUrl: 'https://gw.example.com/v1', + api: 'openai-completions' as const, + models: [{ id: 'my-model' }], + }, + }; + + const loopbackProviders = { + local: { + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions' as const, + models: [{ id: 'my-model' }], + }, + }; + + function fakeReporter() { + return { error: vi.fn() } as unknown as InstanceType; + } + + it('returns false and calls reporter.error when a pi item has a remote provider with no key', () => { + const reporter = fakeReporter(); + const items = [{ runtime: 'pi' as const, providers: remoteProviders }]; + // Explicit empty env keeps the test hermetic regardless of the runner's environment. + const result = verifyCustomProviderAuthForRun(items, reporter, {}); + expect(result).toBe(false); + expect(reporter.error).toHaveBeenCalledOnce(); + expect((reporter.error as ReturnType).mock.calls[0]![0]).toContain('litellm'); + }); + + it('returns true when a pi item has a loopback provider with no key', () => { + const reporter = fakeReporter(); + const items = [{ runtime: 'pi' as const, providers: loopbackProviders }]; + const result = verifyCustomProviderAuthForRun(items, reporter, {}); + expect(result).toBe(true); + expect(reporter.error).not.toHaveBeenCalled(); + }); + + it('returns true and skips auth check for claude runtime items even with remote providers', () => { + const reporter = fakeReporter(); + const items = [{ runtime: 'claude' as const, providers: remoteProviders }]; + const result = verifyCustomProviderAuthForRun(items, reporter, {}); + expect(result).toBe(true); + expect(reporter.error).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/warden/src/cli/main.ts b/packages/warden/src/cli/main.ts index e3f3135a..9ade494d 100644 --- a/packages/warden/src/cli/main.ts +++ b/packages/warden/src/cli/main.ts @@ -10,6 +10,10 @@ import { invalidPiModelSelectorMessage, type InvalidPiModelSelector, } from '../sdk/runtimes/model-selectors.js'; +import { + buildPiProviderOptions, + assertCustomProviderAuth, +} from '../sdk/runtimes/custom-provider.js'; import { mapExtractionErrorCode } from '../sdk/errors.js'; import { aggregateAuxiliaryUsageAttribution, mergeAuxiliaryUsage } from '../sdk/usage.js'; import { resolveSkillAsync, SkillLoaderError } from '../skills/loader.js'; @@ -612,6 +616,7 @@ interface SkillToRun { maxTurns?: number; effort?: SkillRunnerOptions['effort']; runtime?: SkillRunnerOptions['runtime']; + providers?: SkillRunnerOptions['providers']; auxiliaryModel?: string; synthesisModel?: string; auxiliaryMaxRetries?: number; @@ -643,6 +648,7 @@ type SkillRunnerOptionOverrides = Pick< | 'maxTurns' | 'effort' | 'runtime' + | 'providers' | 'auxiliaryModel' | 'synthesisModel' | 'auxiliaryMaxRetries' @@ -709,6 +715,7 @@ export function mergeSkillRunnerOptions( if (overrides.maxTurns !== undefined) merged.maxTurns = overrides.maxTurns; if (overrides.effort !== undefined) merged.effort = overrides.effort; if (overrides.runtime !== undefined) merged.runtime = overrides.runtime; + if (overrides.providers !== undefined) merged.providers = overrides.providers; if (overrides.auxiliaryModel !== undefined) merged.auxiliaryModel = overrides.auxiliaryModel; if (overrides.synthesisModel !== undefined) merged.synthesisModel = overrides.synthesisModel; if (overrides.auxiliaryMaxRetries !== undefined) { @@ -801,6 +808,27 @@ function verifyClaudeAuthForRun(args: { } } +/** + * Verify that every pi-runtime item with a remote custom provider has a resolvable + * API key before analysis starts. Returns false (and emits an error) on first failure. + */ +export function verifyCustomProviderAuthForRun( + items: { runtime?: SkillRunnerOptions['runtime']; providers?: SkillRunnerOptions['providers'] }[], + reporter: Reporter, + env: NodeJS.ProcessEnv = process.env, +): boolean { + for (const item of items) { + if ((item.runtime ?? 'pi') !== 'pi') continue; + try { + assertCustomProviderAuth(buildPiProviderOptions(item.providers, env)); + } catch (error) { + reporter.error(error instanceof Error ? error.message : String(error)); + return false; + } + } + return true; +} + function renderSkillRunHeader(args: { reporter: Reporter; skill: SkillDefinition; @@ -901,20 +929,30 @@ export function resolveCliDefaultModel( ); } -/** Resolve the default auxiliary model used for helper and repair passes. */ +/** + * Resolve the default auxiliary model used for helper and repair passes. + * Falls back to the resolved agent/top-level model so a single configured + * model drives every lane (and self-hosted providers stay self-contained + * instead of escaping to a runtime default on another provider). + */ export function resolveCliDefaultAuxiliaryModel( - config: Pick | null | undefined + config: Pick | null | undefined, + cliModel?: string ): string | undefined { - return emptyToUndefined(config?.defaults?.auxiliary?.model); + return ( + emptyToUndefined(config?.defaults?.auxiliary?.model) ?? + resolveCliDefaultModel(config, cliModel) + ); } /** Resolve the default synthesis model, falling back to the auxiliary lane when unset. */ export function resolveCliDefaultSynthesisModel( - config: Pick | null | undefined + config: Pick | null | undefined, + cliModel?: string ): string | undefined { return ( emptyToUndefined(config?.defaults?.synthesis?.model) ?? - resolveCliDefaultAuxiliaryModel(config) + resolveCliDefaultAuxiliaryModel(config, cliModel) ); } @@ -1072,8 +1110,8 @@ export async function runSkills( const repoPath = findRepoPath(cwd); const config = loadOptionalConfig(options, repoPath); const defaultModel = resolveCliDefaultModel(config, options.model); - const defaultAuxiliaryModel = resolveCliDefaultAuxiliaryModel(config); - const defaultSynthesisModel = resolveCliDefaultSynthesisModel(config); + const defaultAuxiliaryModel = resolveCliDefaultAuxiliaryModel(config, options.model); + const defaultSynthesisModel = resolveCliDefaultSynthesisModel(config, options.model); const defaultEffort = resolveCliEffort(config, options.effort); const defaultRuntime = options.runtime ?? config?.defaults?.runtime ?? 'pi'; const pathToClaudeCodeExecutable = resolveClaudeCodeExecutablePath(); @@ -1098,6 +1136,7 @@ export async function runSkills( maxTurns: match?.maxTurns ?? config?.defaults?.agent?.maxTurns ?? config?.defaults?.maxTurns, effort: options.effort ?? match?.effort ?? defaultEffort, runtime: options.runtime ?? match?.runtime ?? config?.defaults?.runtime ?? 'pi', + providers: match?.providers ?? config?.defaults?.providers, auxiliaryModel: match?.auxiliaryModel ?? defaultAuxiliaryModel, synthesisModel: match?.synthesisModel ?? defaultSynthesisModel, auxiliaryMaxRetries: @@ -1125,6 +1164,7 @@ export async function runSkills( model: t.model, maxTurns: t.maxTurns, runtime: options.runtime ?? t.runtime, + providers: t.providers, effort: options.effort ?? t.effort, auxiliaryModel: t.auxiliaryModel, synthesisModel: t.synthesisModel, @@ -1159,6 +1199,10 @@ export async function runSkills( return 1; } + if (!verifyCustomProviderAuthForRun(skillsToRun, reporter)) { + return 1; + } + // Build skill tasks // Model precedence: defaults.agent.model > defaults.model > CLI flag > WARDEN_MODEL env var > SDK default // sdkModel is undefined when no model is explicitly configured (lets SDK use its default). @@ -1500,6 +1544,14 @@ async function runConfigMode(options: CLIOptions, reporter: Reporter): Promise ({ + runtime: options.runtime ?? trigger.runtime, + providers: trigger.providers, + })); + if (!verifyCustomProviderAuthForRun(customProviderItems, reporter)) { + return 1; + } + // Build trigger tasks const effectiveMinConfidence = options.minConfidence ?? config.defaults?.minConfidence ?? 'medium'; const specs: RunSkillSpec[] = triggersToRun.map((trigger) => ({ @@ -1515,6 +1567,7 @@ async function runConfigMode(options: CLIOptions, reporter: Reporter): Promise { expect(resolved?.auxiliaryModel).toBe('claude-haiku-4-5'); expect(resolved?.synthesisModel).toBe('claude-haiku-4-5'); }); + + it('inherits the top-level model for the auxiliary and synthesis lanes when unset', () => { + const config: WardenConfig = { + ...baseConfig, + defaults: { + model: 'litellm/gemma-4-12b-coder', + }, + }; + + const [resolved] = resolveSkillConfigs(config); + + expect(resolved?.auxiliaryModel).toBe('litellm/gemma-4-12b-coder'); + expect(resolved?.synthesisModel).toBe('litellm/gemma-4-12b-coder'); + }); + + it('prefers explicit auxiliary and synthesis models over the inherited top-level model', () => { + const config: WardenConfig = { + ...baseConfig, + defaults: { + model: 'litellm/gemma-4-12b-coder', + auxiliary: { model: 'litellm/cheap' }, + }, + }; + + const [resolved] = resolveSkillConfigs(config); + + expect(resolved?.auxiliaryModel).toBe('litellm/cheap'); + expect(resolved?.synthesisModel).toBe('litellm/cheap'); + }); }); describe('minConfidence merge', () => { @@ -581,6 +610,19 @@ describe('resolveSkillConfigs', () => { expect(resolved?.minConfidence).toBeUndefined(); }); }); + + it('propagates defaults.providers onto every resolved trigger', () => { + const config = WardenConfigSchema.parse({ + version: 1, + defaults: { providers: { litellm: { baseUrl: 'http://localhost:4000/v1', models: [{ id: 'm' }] } } }, + skills: [{ name: 'a' }, { name: 'b', triggers: [{ type: 'local' }] }], + }); + const resolved = resolveSkillConfigs(config); + expect(resolved).toHaveLength(2); + for (const trigger of resolved) { + expect(trigger.providers?.['litellm']?.baseUrl).toBe('http://localhost:4000/v1'); + } + }); }); describe('mergeWardenConfigs', () => { @@ -899,6 +941,47 @@ describe('resolveLayeredSkillConfigs', () => { expect(resolved[1]?.runtime).toBe('pi'); }); + it('lets repo-defined skills inherit base custom providers when repo defaults omit them', () => { + const providers = { + litellm: { + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions' as const, + models: [{ id: 'gemma', name: 'gemma' }], + }, + }; + const baseConfig: WardenConfig = { + version: 1, + defaults: { + runtime: 'pi', + providers, + }, + skills: [{ + name: 'org-skill', + triggers: [{ type: 'pull_request', actions: ['opened'] }], + }], + }; + + const repoConfig: WardenConfig = { + version: 1, + skills: [{ + name: 'repo-skill', + triggers: [{ type: 'pull_request', actions: ['opened'] }], + }], + }; + + const resolved = resolveLayeredSkillConfigs({ + config: { version: 1, skills: [] }, + baseConfig, + repoConfig, + }); + + expect(resolved).toHaveLength(2); + // Both the base-layer and the repo-layer trigger carry the org custom + // providers; previously the repo layer lost them. + expect(resolved.find((t) => t.name === 'org-skill')?.providers).toEqual(providers); + expect(resolved.find((t) => t.name === 'repo-skill')?.providers).toEqual(providers); + }); + it('lets repo-defined skills inherit base verification defaults when omitted', () => { const baseConfig: WardenConfig = { version: 1, @@ -1644,3 +1727,51 @@ describe('logs config', () => { expect(result.data?.logs).toBeUndefined(); }); }); + +describe('providers config', () => { + const base = { version: 1 as const, skills: [] }; + + it('accepts a valid custom provider', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { + providers: { + litellm: { + baseUrl: 'http://localhost:4000/v1', + models: [{ id: 'my-model' }], + }, + }, + }, + }); + expect(result.success).toBe(true); + if (result.success) { + const p = result.data.defaults?.providers?.['litellm']; + expect(p?.api).toBe('openai-completions'); // default applied + expect(p?.models[0]?.id).toBe('my-model'); + } + }); + + it('rejects a non-URL baseUrl', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { litellm: { baseUrl: 'not a url', models: [{ id: 'm' }] } } }, + }); + expect(result.success).toBe(false); + }); + + it('rejects an unsupported api value', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { x: { baseUrl: 'http://h/v1', api: 'cohere', models: [{ id: 'm' }] } } }, + }); + expect(result.success).toBe(false); + }); + + it('rejects an empty models array', () => { + const result = WardenConfigSchema.safeParse({ + ...base, + defaults: { providers: { x: { baseUrl: 'http://h/v1', models: [] } } }, + }); + expect(result.success).toBe(false); + }); +}); diff --git a/packages/warden/src/config/loader.ts b/packages/warden/src/config/loader.ts index 20e43d5a..c3c8075b 100644 --- a/packages/warden/src/config/loader.ts +++ b/packages/warden/src/config/loader.ts @@ -19,6 +19,7 @@ import { type LogsConfig, type RuntimeName, type AgentRuntimeConfig, + type ProvidersConfig, } from './schema.js'; import type { SeverityThreshold, ConfidenceThreshold } from '../types/index.js'; @@ -179,6 +180,14 @@ function inheritRepoLayerDefaults(base?: Defaults, repo?: Defaults): Defaults | inherited.runtime = base.runtime; } + // Inherit the org base custom providers, like runtime, as an execution- + // environment default: a repo layer that only adds skills should still reach + // the providers the org defined. Per-skill policy defaults (model, failOn, + // ignorePaths, ...) intentionally do not cross layers. + if (base?.providers !== undefined && inherited.providers === undefined) { + inherited.providers = base.providers; + } + const verification = mergeNestedConfig(base?.verification, repo?.verification); if (verification) { inherited.verification = verification; @@ -399,6 +408,8 @@ export interface ResolvedTrigger { effort?: AgentRuntimeConfig['effort']; /** Runtime backend for all model-backed execution. */ runtime?: RuntimeName; + /** Custom OpenAI-compatible providers for the Pi runtime. */ + providers?: ProvidersConfig; /** Model for auxiliary structured model calls. */ auxiliaryModel?: string; /** Model for post-analysis synthesis/consolidation. */ @@ -493,7 +504,19 @@ export function resolveSkillConfigs( const envModel = emptyToUndefined(process.env['WARDEN_MODEL']); const result: ResolvedTrigger[] = []; const runtime = defaults?.runtime ?? 'pi'; - const auxiliaryModel = emptyToUndefined(defaults?.auxiliary?.model); + const providers = defaults?.providers; + // Default agent/top-level model, used as the fallback for the auxiliary and + // synthesis lanes so one configured model drives every lane (and self-hosted + // providers stay self-contained instead of escaping to a runtime default on + // another provider). Explicit auxiliary/synthesis models still win. + const defaultAgentModel = + emptyToUndefined(defaults?.agent?.model) ?? + emptyToUndefined(defaults?.model) ?? + emptyToUndefined(cliModel) ?? + envModel; + const auxiliaryModel = + emptyToUndefined(defaults?.auxiliary?.model) ?? + defaultAgentModel; const synthesisModel = emptyToUndefined(defaults?.synthesis?.model) ?? auxiliaryModel; @@ -544,6 +567,7 @@ export function resolveSkillConfigs( maxTurns: baseMaxTurns, effort, runtime, + providers, auxiliaryModel, synthesisModel, auxiliaryMaxRetries, @@ -579,6 +603,7 @@ export function resolveSkillConfigs( maxTurns: trigger.maxTurns ?? baseMaxTurns, effort, runtime, + providers, auxiliaryModel, synthesisModel, auxiliaryMaxRetries, diff --git a/packages/warden/src/config/schema.ts b/packages/warden/src/config/schema.ts index 245c30df..84563b4e 100644 --- a/packages/warden/src/config/schema.ts +++ b/packages/warden/src/config/schema.ts @@ -216,6 +216,50 @@ export const DEFAULT_SCAN_LIMITS: Required = { maxFileLines: 3_000, }; +// Per-model definition for a custom provider. Only `id` is required; Warden +// fills the remaining fields Pi requires with sensible defaults. +export const ProviderModelConfigSchema = z.object({ + /** Model id as exposed by the endpoint (e.g. the LiteLLM model name). */ + id: z.string().min(1), + /** Display name. Defaults to `id`. */ + name: z.string().min(1).optional(), + /** Whether the model supports reasoning/thinking. Default: false. */ + reasoning: z.boolean().optional(), + /** Accepted input modalities. Default: ["text"]. */ + input: z.array(z.enum(['text', 'image'])).min(1).optional(), + /** Context window in tokens. Default: 128000. */ + contextWindow: z.number().int().positive().optional(), + /** Max output tokens. Default: 8192. */ + maxTokens: z.number().int().positive().optional(), + /** Per-token cost (USD per token). Default: all zeros. */ + cost: z.object({ + input: z.number().nonnegative(), + output: z.number().nonnegative(), + cacheRead: z.number().nonnegative(), + cacheWrite: z.number().nonnegative(), + }).optional(), +}).strict(); +export type ProviderModelConfig = z.infer; + +// A custom, self-hosted or gateway provider (e.g. LiteLLM). OpenAI-compatible only in v1. +export const ProviderConfigSchema = z.object({ + /** OpenAI-compatible base URL, typically ending in /v1. */ + baseUrl: z.string().url(), + /** Wire protocol. Only "openai-completions" is supported today. */ + api: z.enum(['openai-completions']).default('openai-completions'), + /** Extra HTTP headers sent on every request. */ + headers: z.record(z.string(), z.string()).optional(), + /** Env var holding the API key. Defaults to WARDEN__API_KEY then _API_KEY. */ + apiKeyEnv: z.string().min(1).optional(), + /** Models this provider exposes. At least one is required. */ + models: z.array(ProviderModelConfigSchema).min(1), +}).strict(); +export type ProviderConfig = z.infer; + +// Map of provider name -> provider config. The name becomes the model-selector prefix. +export const ProvidersConfigSchema = z.record(z.string().min(1), ProviderConfigSchema); +export type ProvidersConfig = z.infer; + // Default configuration that skills inherit from export const DefaultsSchema = z.object({ /** Fail the build when findings meet this severity */ @@ -255,6 +299,8 @@ export const DefaultsSchema = z.object({ ignore: IgnoreConfigSchema.optional(), /** Global scan limits applied after ignore filtering */ scan: ScanConfigSchema.optional(), + /** Custom OpenAI-compatible providers (e.g. self-hosted LiteLLM). Keyed by provider name. */ + providers: ProvidersConfigSchema.optional(), /** Delay in milliseconds between batch starts when processing files in parallel. Default: 0 */ batchDelayMs: z.number().int().nonnegative().optional(), /** Max retries for auxiliary structured model calls (extraction repair, merging, dedup, fix evaluation). Default: 5 */ diff --git a/packages/warden/src/output/dedup.ts b/packages/warden/src/output/dedup.ts index eca21645..de0b168b 100644 --- a/packages/warden/src/output/dedup.ts +++ b/packages/warden/src/output/dedup.ts @@ -3,7 +3,7 @@ import type { Octokit } from '@octokit/rest'; import { z } from 'zod'; import type { Confidence, Finding, Severity, UsageStats } from '../types/index.js'; import { findingLine } from '../types/index.js'; -import { getRuntime } from '../sdk/runtimes/index.js'; +import { getRuntime, getRuntimeProviderOptions } from '../sdk/runtimes/index.js'; import { applyMergeGroups, canUseRuntimeAuth } from '../sdk/extract.js'; import type { AuxiliaryCallOptions } from '../sdk/extract.js'; import { @@ -581,7 +581,7 @@ async function findSemanticDuplicates( findings: Finding[], existingComments: ExistingComment[], apiKey: string | undefined, - options: Pick = {} + options: Pick = {} ): Promise { if (findings.length === 0 || existingComments.length === 0) { return { matches: new Map() }; @@ -622,6 +622,7 @@ Return [] if none are duplicates.`), model: options.model, maxTokens: 512, maxRetries: options.maxRetries, + providerOptions: getRuntimeProviderOptions(options.runtime ?? 'claude', { providers: options.providers }), }); if (!result.success) { @@ -925,6 +926,7 @@ Singletons (findings with no duplicates) should not appear in any group. model: options.model, maxTokens: 512, maxRetries: options.maxRetries, + providerOptions: getRuntimeProviderOptions(options.runtime ?? 'claude', { providers: options.providers }), }); if (!result.success) { diff --git a/packages/warden/src/sdk/analyze.ts b/packages/warden/src/sdk/analyze.ts index 14bcc5a1..265dfe29 100644 --- a/packages/warden/src/sdk/analyze.ts +++ b/packages/warden/src/sdk/analyze.ts @@ -164,6 +164,7 @@ async function parseHunkOutput( const fallback = await extractFindingsWithLLM(result.text, { apiKey: options.apiKey, runtime: options.runtime, + providers: options.providers, model: options.auxiliaryModel, maxRetries: options.auxiliaryMaxRetries, agentName: skillName, @@ -396,6 +397,7 @@ async function analyzeHunk( }, providerOptions: getRuntimeProviderOptions(runtimeName, { pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable, + providers: options.providers, }), })); @@ -1159,6 +1161,7 @@ async function runSkillAnalysis( repoPath: context.repoPath, apiKey: options.apiKey, runtime: options.runtime, + providers: options.providers, auxiliaryModel: options.auxiliaryModel, synthesisModel: options.synthesisModel, auxiliaryMaxRetries: options.auxiliaryMaxRetries, diff --git a/packages/warden/src/sdk/extract.test.ts b/packages/warden/src/sdk/extract.test.ts index 5c9bbdc6..11c9d69e 100644 --- a/packages/warden/src/sdk/extract.test.ts +++ b/packages/warden/src/sdk/extract.test.ts @@ -10,18 +10,43 @@ import { canUseRuntimeAuth, } from './extract.js'; import type { Finding } from '../types/index.js'; +import { getRuntime } from './runtimes/index.js'; +import type { Runtime } from './runtimes/index.js'; + +// Mock runtimes to avoid real API calls +vi.mock('./runtimes/index.js', () => ({ + getRuntime: vi.fn(), + getRuntimeProviderOptions: vi.fn((_name: string, opts: { providers?: unknown }) => { + if (!opts?.providers) return undefined; + // Minimal simulation: return providerOptions shape used by Pi adapter + const providers = opts.providers as Record; + return { + providers: Object.entries(providers).map(([name]) => ({ name })), + }; + }), +})); -// Mock callHaiku to avoid real API calls -vi.mock('./haiku.js', async (importOriginal) => { - const actual: Record = await importOriginal(); +function makeSynthesisRuntime(data: unknown): Runtime { return { - ...actual, - callHaiku: vi.fn(), - }; -}); + name: 'claude' as const, + runSkill: vi.fn(), + runAuxiliary: vi.fn(), + runSynthesis: vi.fn().mockResolvedValue({ + success: true, + data, + usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, + }), + } as unknown as Runtime; +} -import { callHaiku } from './haiku.js'; -const mockCallHaiku = vi.mocked(callHaiku); +function makeAuxiliaryRuntime(result: unknown): Runtime { + return { + name: 'pi' as const, + runSkill: vi.fn(), + runAuxiliary: vi.fn().mockResolvedValue(result), + runSynthesis: vi.fn(), + } as unknown as Runtime; +} function makeFinding(overrides: Partial = {}): Finding { return { @@ -45,11 +70,12 @@ describe('extractFindingsWithLLM', () => { }); it('preserves the LLM extraction failure prefix for stable error classification', async () => { - mockCallHaiku.mockResolvedValue({ + const runtime = makeAuxiliaryRuntime({ success: false, error: 'Request timed out', usage: { inputTokens: 10, outputTokens: 0, costUSD: 0.001 }, }); + vi.mocked(getRuntime).mockReturnValue(runtime); const result = await extractFindingsWithLLM('{ "findings": [', { apiKey: 'test-key' }); @@ -60,6 +86,38 @@ describe('extractFindingsWithLLM', () => { usage: { inputTokens: 10, outputTokens: 0, costUSD: 0.001 }, }); }); + + it('forwards providerOptions built from providers to the auxiliary runtime', async () => { + const runAuxiliaryMock = vi.fn().mockResolvedValue({ + success: true, + data: { findings: [] }, + usage: { inputTokens: 10, outputTokens: 5, costUSD: 0.001 }, + }); + const runtime: Runtime = { + name: 'pi', + runSkill: vi.fn(), + runAuxiliary: runAuxiliaryMock, + runSynthesis: vi.fn(), + } as unknown as Runtime; + vi.mocked(getRuntime).mockReturnValue(runtime); + + const rawText = '{ "findings": [] }'; + await extractFindingsWithLLM(rawText, { + runtime: 'pi', + providers: { + litellm: { + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions', + models: [{ id: 'm' }], + }, + }, + }); + + const req = runAuxiliaryMock.mock.calls[0]?.[0]; + expect(req).toBeDefined(); + expect(req.providerOptions).toBeDefined(); + expect(req.providerOptions.providers[0].name).toBe('litellm'); + }); }); describe('deduplicateFindings', () => { @@ -134,11 +192,8 @@ describe('mergeCrossLocationFindings', () => { makeFinding({ id: 'f2', title: 'Issue B', location: { path: 'src/b.ts', startLine: 1 } }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + const runtime = makeSynthesisRuntime([]); + vi.mocked(getRuntime).mockReturnValue(runtime); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', @@ -147,7 +202,7 @@ describe('mergeCrossLocationFindings', () => { }); expect(result.findings).toHaveLength(2); expect(result.mergedCount).toBe(0); - expect(mockCallHaiku).toHaveBeenCalledWith(expect.objectContaining({ + expect(vi.mocked(runtime.runSynthesis)).toHaveBeenCalledWith(expect.objectContaining({ model: 'claude-test-fast', })); }); @@ -168,11 +223,7 @@ describe('mergeCrossLocationFindings', () => { }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); const onFindingProcessing = vi.fn(); const result = await mergeCrossLocationFindings(findings, { @@ -203,11 +254,7 @@ describe('mergeCrossLocationFindings', () => { makeFinding({ id: 'f3', severity: 'medium', location: { path: 'src/a.ts', startLine: 5 } }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2, 3]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2, 3]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); expect(result.findings).toHaveLength(1); @@ -233,11 +280,7 @@ describe('mergeCrossLocationFindings', () => { }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); // Same severity, same confidence, a.ts < b.ts alphabetically @@ -259,11 +302,7 @@ describe('mergeCrossLocationFindings', () => { }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); expect(result.findings[0]!.additionalLocations).toEqual([ @@ -279,11 +318,7 @@ describe('mergeCrossLocationFindings', () => { makeFinding({ id: 'f3', location: { path: 'src/b.ts', startLine: 2 } }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], // groups the two with-location findings (indices 1,2 from withLocations array) - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); // f1 absorbs f3, f2 (no location) passes through @@ -297,11 +332,16 @@ describe('mergeCrossLocationFindings', () => { makeFinding({ id: 'f2', location: { path: 'src/b.ts', startLine: 2 } }), ]; - mockCallHaiku.mockResolvedValue({ - success: false, - error: 'API error', - usage: { inputTokens: 100, outputTokens: 0, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue({ + name: 'claude', + runSkill: vi.fn(), + runAuxiliary: vi.fn(), + runSynthesis: vi.fn().mockResolvedValue({ + success: false, + error: 'API error', + usage: { inputTokens: 100, outputTokens: 0, costUSD: 0.001 }, + }), + } as unknown as Runtime); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); expect(result.findings).toHaveLength(2); @@ -323,11 +363,7 @@ describe('mergeCrossLocationFindings', () => { location: { path: 'src/b.ts', startLine: 2 }, }); - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); await mergeCrossLocationFindings([f1, f2], { apiKey: 'test-key', repoPath: tempDir }); // Original f1 should NOT have additionalLocations added @@ -342,11 +378,7 @@ describe('mergeCrossLocationFindings', () => { ]; // LLM returns overlapping groups: [1,2] and [2,3] - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2], [2, 3]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2], [2, 3]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); // f2 is absorbed by group [1,2]. Group [2,3] should skip f2 (already absorbed), @@ -372,11 +404,7 @@ describe('mergeCrossLocationFindings', () => { }), ]; - mockCallHaiku.mockResolvedValue({ - success: true, - data: [[1, 2]], - usage: { inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, - }); + vi.mocked(getRuntime).mockReturnValue(makeSynthesisRuntime([[1, 2]])); const result = await mergeCrossLocationFindings(findings, { apiKey: 'test-key', repoPath: tempDir }); // src/b.ts:2 should only appear once in additionalLocations (deduped) diff --git a/packages/warden/src/sdk/extract.ts b/packages/warden/src/sdk/extract.ts index b7ef0229..ac9d00b9 100644 --- a/packages/warden/src/sdk/extract.ts +++ b/packages/warden/src/sdk/extract.ts @@ -4,8 +4,9 @@ import { z } from 'zod'; import { customAlphabet } from 'nanoid'; import { FindingSchema, compareFindingPriority } from '../types/index.js'; import type { Finding, Location, UsageStats } from '../types/index.js'; -import { getRuntime } from './runtimes/index.js'; +import { getRuntime, getRuntimeProviderOptions } from './runtimes/index.js'; import type { RuntimeName } from './runtimes/index.js'; +import type { ProvidersConfig } from '../config/schema.js'; import type { FindingProcessingEvent } from './types.js'; import { buildJsonOutputSection, @@ -29,6 +30,8 @@ export type ExtractFindingsResult = export interface AuxiliaryCallOptions { apiKey?: string; runtime?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; model?: string; maxRetries?: number; agentName?: string; @@ -237,6 +240,7 @@ If no findings exist, return: {"findings": []}`), maxTokens: LLM_FALLBACK_MAX_TOKENS, timeout: LLM_FALLBACK_TIMEOUT_MS, maxRetries: options.maxRetries, + providerOptions: getRuntimeProviderOptions(runtimeName, { providers: options.providers }), }); if (!result.success) { @@ -556,6 +560,7 @@ Singletons should not appear. Return [] if no findings describe the same issue.` model: options?.model, maxTokens: 512, maxRetries: options?.maxRetries, + providerOptions: getRuntimeProviderOptions(options?.runtime ?? 'claude', { providers: options?.providers }), }); if (!result.success) { diff --git a/packages/warden/src/sdk/json-output.ts b/packages/warden/src/sdk/json-output.ts index e0279644..01c84300 100644 --- a/packages/warden/src/sdk/json-output.ts +++ b/packages/warden/src/sdk/json-output.ts @@ -3,7 +3,8 @@ import type { UsageStats } from '../types/index.js'; import { extractJson } from './haiku.js'; import { canUseRuntimeAuth } from './extract.js'; import { buildJsonOutputSection, buildTaggedSection, joinPromptSections } from './prompt-sections.js'; -import { getRuntime, type Runtime, type RuntimeName } from './runtimes/index.js'; +import { getRuntime, getRuntimeProviderOptions, type Runtime, type RuntimeName } from './runtimes/index.js'; +import type { ProvidersConfig } from '../config/schema.js'; const JSON_REPAIR_MAX_CHARS = 60_000; const JSON_REPAIR_MAX_TOKENS = 16_384; @@ -18,6 +19,8 @@ export interface JsonOutputRepairOptions { agentName?: string; runtime?: Runtime; runtimeName?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; model?: string; maxRetries?: number; maxTokens?: number; @@ -86,6 +89,7 @@ async function repairJsonOutput( maxTokens: repair.maxTokens ?? JSON_REPAIR_MAX_TOKENS, timeout: repair.timeout ?? JSON_REPAIR_TIMEOUT_MS, schema, + providerOptions: getRuntimeProviderOptions(runtime.name, { providers: repair.providers }), prompt: joinPromptSections([ ` Extract and repair the JSON value from this model output. diff --git a/packages/warden/src/sdk/post-process.ts b/packages/warden/src/sdk/post-process.ts index 1660ab93..d670d4c2 100644 --- a/packages/warden/src/sdk/post-process.ts +++ b/packages/warden/src/sdk/post-process.ts @@ -1,4 +1,4 @@ -import type { Effort, SkillDefinition } from '../config/schema.js'; +import type { Effort, ProvidersConfig, SkillDefinition } from '../config/schema.js'; import { emitDedupMetrics } from '../sentry.js'; import type { Finding } from '../types/index.js'; import { deduplicateFindings, mergeCrossLocationFindings } from './extract.js'; @@ -12,6 +12,8 @@ export interface PostProcessFindingsOptions { repoPath: string; apiKey?: string; runtime?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; auxiliaryModel?: string; synthesisModel?: string; auxiliaryMaxRetries?: number; @@ -48,6 +50,7 @@ export async function postProcessFindings( skill: options.skill, apiKey: options.apiKey, runtime: options.runtime, + providers: options.providers, model: options.auxiliaryModel, maxTurns: options.maxTurns, effort: options.effort, @@ -71,6 +74,7 @@ export async function postProcessFindings( apiKey: options.apiKey, repoPath: options.repoPath, runtime: options.runtime, + providers: options.providers, model: options.synthesisModel, maxRetries: options.auxiliaryMaxRetries, agentName: options.skill.name, diff --git a/packages/warden/src/sdk/runtimes/custom-provider.test.ts b/packages/warden/src/sdk/runtimes/custom-provider.test.ts new file mode 100644 index 00000000..22b0b415 --- /dev/null +++ b/packages/warden/src/sdk/runtimes/custom-provider.test.ts @@ -0,0 +1,74 @@ +import { describe, it, expect } from 'vitest'; +import { + buildPiProviderOptions, + resolveProviderApiKey, + isLoopbackBaseUrl, + assertCustomProviderAuth, +} from './custom-provider.js'; +import type { ProvidersConfig } from '../../config/schema.js'; + +const providers: ProvidersConfig = { + litellm: { baseUrl: 'https://gw.example.com/v1', api: 'openai-completions', models: [{ id: 'my-model' }] }, +}; + +describe('resolveProviderApiKey', () => { + it('prefers apiKeyEnv when set', () => { + expect(resolveProviderApiKey('litellm', 'MY_KEY', { MY_KEY: 'k1', WARDEN_LITELLM_API_KEY: 'k2' })).toBe('k1'); + }); + it('falls back to WARDEN__API_KEY then _API_KEY', () => { + expect(resolveProviderApiKey('litellm', undefined, { WARDEN_LITELLM_API_KEY: 'k2' })).toBe('k2'); + expect(resolveProviderApiKey('litellm', undefined, { LITELLM_API_KEY: 'k3' })).toBe('k3'); + }); + it('returns undefined when nothing is set', () => { + expect(resolveProviderApiKey('litellm', undefined, {})).toBeUndefined(); + }); +}); + +describe('buildPiProviderOptions', () => { + it('returns undefined for empty input', () => { + expect(buildPiProviderOptions(undefined, {})).toBeUndefined(); + expect(buildPiProviderOptions({}, {})).toBeUndefined(); + }); + it('applies model defaults and resolves the key', () => { + const built = buildPiProviderOptions(providers, { WARDEN_LITELLM_API_KEY: 'k2' }); + const p = built?.providers[0]; + expect(p?.name).toBe('litellm'); + expect(p?.apiKey).toBe('k2'); + const m = p?.models[0]; + expect(m).toMatchObject({ + id: 'my-model', + name: 'my-model', + api: 'openai-completions', + reasoning: false, + input: ['text'], + contextWindow: 128000, + maxTokens: 8192, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }); + }); +}); + +describe('isLoopbackBaseUrl', () => { + it('recognizes loopback hosts', () => { + expect(isLoopbackBaseUrl('http://localhost:4000/v1')).toBe(true); + expect(isLoopbackBaseUrl('http://127.0.0.1:4000')).toBe(true); + expect(isLoopbackBaseUrl('https://gw.example.com/v1')).toBe(false); + }); + it('recognizes bracketed IPv6 loopback', () => { + expect(isLoopbackBaseUrl('http://[::1]:4000/v1')).toBe(true); + }); +}); + +describe('assertCustomProviderAuth', () => { + it('throws for a non-loopback provider without a key', () => { + const built = buildPiProviderOptions(providers, {}); + expect(() => assertCustomProviderAuth(built)).toThrow(/litellm/); + }); + it('passes for a loopback provider without a key', () => { + const built = buildPiProviderOptions( + { local: { baseUrl: 'http://localhost:4000/v1', api: 'openai-completions', models: [{ id: 'm' }] } }, + {}, + ); + expect(() => assertCustomProviderAuth(built)).not.toThrow(); + }); +}); diff --git a/packages/warden/src/sdk/runtimes/custom-provider.ts b/packages/warden/src/sdk/runtimes/custom-provider.ts new file mode 100644 index 00000000..51dd056a --- /dev/null +++ b/packages/warden/src/sdk/runtimes/custom-provider.ts @@ -0,0 +1,107 @@ +/** + * Normalization and auth resolution for custom OpenAI-compatible providers + * (e.g. self-hosted LiteLLM). Pure functions over a passed-in env object so + * they are trivially testable and never read process.env implicitly. + */ +import type { ProvidersConfig } from '../../config/schema.js'; + +const DEFAULT_CONTEXT_WINDOW = 128_000; +const DEFAULT_MAX_TOKENS = 8_192; +const DEFAULT_COST = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } as const; + +function sanitizeProviderName(name: string): string { + return name.toUpperCase().replace(/[^A-Z0-9]/g, '_'); +} + +export interface PiProviderModel { + id: string; + name: string; + api: 'openai-completions'; + reasoning: boolean; + input: ('text' | 'image')[]; + cost: { input: number; output: number; cacheRead: number; cacheWrite: number }; + contextWindow: number; + maxTokens: number; +} + +export interface PiProvider { + name: string; + baseUrl: string; + api: 'openai-completions'; + headers?: Record; + apiKey?: string; + models: PiProviderModel[]; +} + +export type PiProviderOptions = { providers: PiProvider[] } | undefined; + +/** Resolve a provider API key from the environment. apiKeyEnv wins, then conventional names. */ +export function resolveProviderApiKey( + name: string, + apiKeyEnv: string | undefined, + env: NodeJS.ProcessEnv, +): string | undefined { + const upper = sanitizeProviderName(name); + const candidates = apiKeyEnv ? [apiKeyEnv] : [`WARDEN_${upper}_API_KEY`, `${upper}_API_KEY`]; + for (const candidate of candidates) { + const value = env[candidate]; + if (value) return value; + } + return undefined; +} + +/** Normalize warden.toml providers into Pi registerProvider input with defaults + resolved keys. */ +export function buildPiProviderOptions( + providers: ProvidersConfig | undefined, + env: NodeJS.ProcessEnv, +): PiProviderOptions { + if (!providers) return undefined; + const entries = Object.entries(providers); + if (entries.length === 0) return undefined; + + const built: PiProvider[] = entries.map(([name, config]) => ({ + name, + baseUrl: config.baseUrl, + api: config.api, + ...(config.headers ? { headers: config.headers } : {}), + apiKey: resolveProviderApiKey(name, config.apiKeyEnv, env), + models: config.models.map((model) => ({ + id: model.id, + name: model.name ?? model.id, + api: config.api, + reasoning: model.reasoning ?? false, + input: model.input ?? ['text'], + cost: model.cost ?? { ...DEFAULT_COST }, + contextWindow: model.contextWindow ?? DEFAULT_CONTEXT_WINDOW, + maxTokens: model.maxTokens ?? DEFAULT_MAX_TOKENS, + })), + })); + + return { providers: built }; +} + +/** True when the base URL points at a loopback host (unauthenticated runs allowed). */ +export function isLoopbackBaseUrl(baseUrl: string): boolean { + try { + const host = new URL(baseUrl).hostname; + // Node's WHATWG URL returns IPv6 hosts bracketed (e.g. `[::1]`); the + // unbracketed `::1` is kept as a defensive fallback for pre-normalized input. + return host === 'localhost' || host === '127.0.0.1' || host === '[::1]' || host === '::1'; + } catch { + return false; + } +} + +/** Fail fast when a non-loopback provider has no resolvable API key. */ +export function assertCustomProviderAuth(options: PiProviderOptions): void { + if (!options) return; + for (const provider of options.providers) { + if (!provider.apiKey && !isLoopbackBaseUrl(provider.baseUrl)) { + throw new Error( + `Custom provider "${provider.name}" has no API key. ` + + `Set WARDEN_${sanitizeProviderName(provider.name)}_API_KEY ` + + `(or the configured apiKeyEnv), or use a localhost baseUrl for an unauthenticated endpoint.`, + ); + } + } +} diff --git a/packages/warden/src/sdk/runtimes/index.test.ts b/packages/warden/src/sdk/runtimes/index.test.ts index 4ed4c278..9d38cbd9 100644 --- a/packages/warden/src/sdk/runtimes/index.test.ts +++ b/packages/warden/src/sdk/runtimes/index.test.ts @@ -7,6 +7,25 @@ import { piRuntime, } from './index.js'; +describe('getRuntimeProviderOptions pi providers', () => { + it('builds pi provider options from config', () => { + const result = getRuntimeProviderOptions('pi', { + providers: { litellm: { baseUrl: 'http://localhost:4000/v1', api: 'openai-completions', models: [{ id: 'm' }] } }, + }) as { providers: { name: string; models: { id: string }[] }[] } | undefined; + expect(result?.providers[0]?.name).toBe('litellm'); + expect(result?.providers[0]?.models[0]?.id).toBe('m'); + }); + + it('returns undefined for pi without providers', () => { + expect(getRuntimeProviderOptions('pi', {})).toBeUndefined(); + }); + + it('still returns the claude executable path', () => { + expect(getRuntimeProviderOptions('claude', { pathToClaudeCodeExecutable: '/bin/claude' })) + .toEqual({ pathToClaudeCodeExecutable: '/bin/claude' }); + }); +}); + describe('runtimes', () => { it('exposes Pi as the default runtime provider', () => { const runtime = getRuntime(); diff --git a/packages/warden/src/sdk/runtimes/index.ts b/packages/warden/src/sdk/runtimes/index.ts index 93a53996..94a1cee7 100644 --- a/packages/warden/src/sdk/runtimes/index.ts +++ b/packages/warden/src/sdk/runtimes/index.ts @@ -1,6 +1,8 @@ import { claudeRuntime } from './claude.js'; import { piRuntime } from './pi.js'; import type { Runtime, RuntimeName } from './types.js'; +import { buildPiProviderOptions } from './custom-provider.js'; +import type { ProvidersConfig } from '../../config/schema.js'; const RUNTIMES: Partial> = { claude: claudeRuntime, @@ -36,6 +38,7 @@ export function getRuntime(name: RuntimeName = 'pi'): Runtime { export interface RuntimeProviderOptionsInput { pathToClaudeCodeExecutable?: string; + providers?: ProvidersConfig; } /** @@ -49,5 +52,12 @@ export function getRuntimeProviderOptions( return { pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable }; } + if (name === 'pi') { + // Resolve provider API keys from the live process env at the runtime + // boundary. Preflight (verifyCustomProviderAuthForRun) resolves against the + // same env, so both phases agree as long as env is not mutated mid-run. + return buildPiProviderOptions(options.providers, process.env); + } + return undefined; } diff --git a/packages/warden/src/sdk/runtimes/pi.test.ts b/packages/warden/src/sdk/runtimes/pi.test.ts index 21585ba3..3fe29d0a 100644 --- a/packages/warden/src/sdk/runtimes/pi.test.ts +++ b/packages/warden/src/sdk/runtimes/pi.test.ts @@ -25,6 +25,7 @@ const piMocks = vi.hoisted(() => { const registry = { find: vi.fn((_provider: string, _modelId: string) => model), getAll: vi.fn(() => [model]), + registerProvider: vi.fn(), }; const session = { sessionId: 'pi-session-1', @@ -540,4 +541,33 @@ describe('piRuntime structured calls', () => { expect(result.error).toContain('Validation failed'); } }); + + it('registers custom providers from providerOptions before resolving the model', async () => { + await piRuntime.runAuxiliary({ + task: 'extraction', + apiKey: undefined, + prompt: 'hi', + schema: z.object({ ok: z.boolean() }), + model: 'litellm/my-model', + providerOptions: { + providers: [{ + name: 'litellm', + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions', + apiKey: 'k', + models: [{ + id: 'my-model', name: 'my-model', api: 'openai-completions', reasoning: false, + input: ['text'], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, maxTokens: 8192, + }], + }], + }, + }); + + expect(piMocks.registry.registerProvider).toHaveBeenCalledWith( + 'litellm', + expect.objectContaining({ baseUrl: 'http://localhost:4000/v1', apiKey: 'k' }), + ); + expect(piMocks.authStorage.setRuntimeApiKey).toHaveBeenCalledWith('litellm', 'k'); + }); }); diff --git a/packages/warden/src/sdk/runtimes/pi.ts b/packages/warden/src/sdk/runtimes/pi.ts index 540b1601..1a60128a 100644 --- a/packages/warden/src/sdk/runtimes/pi.ts +++ b/packages/warden/src/sdk/runtimes/pi.ts @@ -51,6 +51,7 @@ import { setGenAiUsageAttrs, } from '../otel.js'; import { aggregateUsage, emptyUsage } from '../usage.js'; +import type { PiProviderOptions } from './custom-provider.js'; import { InvalidPiModelSelectorError, isPiModelSelector } from './model-selectors.js'; import type { AuxiliaryRunRequest, @@ -102,6 +103,8 @@ interface PiPromptOptions { agentName?: string; model?: string; legacyAnthropicApiKey?: string; + /** Custom providers to register on the model registry before resolving the model. */ + customProviders?: PiProviderOptions; toolNames: string[]; customTools?: ToolDefinition[]; maxTurns?: number; @@ -150,6 +153,26 @@ function createAuthStorage(model: string | undefined, legacyAnthropicApiKey: str return authStorage; } +function registerCustomProviders( + modelRegistry: ModelRegistry, + authStorage: AuthStorage, + customProviders: PiProviderOptions, +): void { + if (!customProviders) return; + for (const provider of customProviders.providers) { + modelRegistry.registerProvider(provider.name, { + baseUrl: provider.baseUrl, + api: provider.api, + ...(provider.headers ? { headers: provider.headers } : {}), + ...(provider.apiKey ? { apiKey: provider.apiKey } : {}), + models: provider.models, + }); + if (provider.apiKey) { + authStorage.setRuntimeApiKey(provider.name, provider.apiKey); + } + } +} + function resolvePiModel( model: string | undefined, registry: ModelRegistry, @@ -345,6 +368,7 @@ async function runPiPrompt(options: PiPromptOptions): Promise { bridgeWardenProviderApiKeyEnv(); const authStorage = createAuthStorage(options.model, options.legacyAnthropicApiKey); const modelRegistry = ModelRegistry.create(authStorage); + registerCustomProviders(modelRegistry, authStorage, options.customProviders); const model = resolvePiModel(options.model, modelRegistry); const settingsManager = buildSettingsManager(options.timeout, options.maxRetries); const agentDir = getAgentDir(); @@ -655,6 +679,7 @@ async function runStructured( tools?: AuxiliaryTool[]; executeTool?: (name: string, input: Record) => Promise; maxIterations?: number; + providerOptions?: unknown; } ): Promise> { const customTools = toPiCustomTools(request.tools, request.executeTool); @@ -687,6 +712,7 @@ async function runStructured( agentName: request.agentName, model: request.model, legacyAnthropicApiKey: request.apiKey, + customProviders: request.providerOptions as PiProviderOptions, toolNames, customTools, toolDescriptions, @@ -758,6 +784,7 @@ export const piRuntime: Runtime = { skillName, tools, allowMutatingTools, + providerOptions, } = request; const { maxTurns = 50, model, effort, abortController } = options; const skillTools = resolvePiSkillTools(tools, allowMutatingTools); @@ -787,6 +814,7 @@ export const piRuntime: Runtime = { agentName: skillName, model, legacyAnthropicApiKey: apiKey, + customProviders: providerOptions as PiProviderOptions, toolNames: skillTools.toolNames, maxTurns, effort, diff --git a/packages/warden/src/sdk/runtimes/types.ts b/packages/warden/src/sdk/runtimes/types.ts index e1a16eb5..e2ec9721 100644 --- a/packages/warden/src/sdk/runtimes/types.ts +++ b/packages/warden/src/sdk/runtimes/types.ts @@ -110,6 +110,8 @@ interface AuxiliaryRunRequestBase { maxTokens?: number; timeout?: number; maxRetries?: number; + /** Provider-specific settings consumed only by the selected runtime adapter. */ + providerOptions?: unknown; } interface AuxiliaryRunRequestWithoutTools extends AuxiliaryRunRequestBase { @@ -137,6 +139,8 @@ export interface SynthesisRunRequest { maxTokens?: number; timeout?: number; maxRetries?: number; + /** Provider-specific settings consumed only by the selected runtime adapter. */ + providerOptions?: unknown; } export interface Runtime { diff --git a/packages/warden/src/sdk/types.ts b/packages/warden/src/sdk/types.ts index 7c773562..206fc7b7 100644 --- a/packages/warden/src/sdk/types.ts +++ b/packages/warden/src/sdk/types.ts @@ -1,6 +1,6 @@ import type { Finding, UsageStats, SkippedFile, RetryConfig, ErrorCode, HunkFailure, HunkTrace } from '../types/index.js'; import type { HunkWithContext } from '../diff/index.js'; -import type { ChunkingConfig, Effort, IgnoreConfig, ScanConfig } from '../config/schema.js'; +import type { ChunkingConfig, Effort, IgnoreConfig, ProvidersConfig, ScanConfig } from '../config/schema.js'; import type { RuntimeName } from './runtimes/index.js'; import type { ProviderFailureCircuitBreaker } from './circuit-breaker.js'; @@ -114,6 +114,8 @@ export interface SkillRunnerOptions { effort?: Effort; /** Runtime backend for all model-backed execution. Defaults to Pi. */ runtime?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; /** Model to use for auxiliary structured model calls. Uses runtime default if not specified. */ auxiliaryModel?: string; /** Model to use for post-analysis synthesis/consolidation. Falls back to auxiliaryModel when not specified. */ diff --git a/packages/warden/src/sdk/verify.ts b/packages/warden/src/sdk/verify.ts index 440ead65..43f5f18c 100644 --- a/packages/warden/src/sdk/verify.ts +++ b/packages/warden/src/sdk/verify.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { Effort, SkillDefinition } from '../config/schema.js'; +import type { Effort, ProvidersConfig, SkillDefinition } from '../config/schema.js'; import { FindingSchema, type Finding, type UsageStats } from '../types/index.js'; import { aggregateUsage } from './usage.js'; import { extractBalancedJson } from './extract.js'; @@ -32,6 +32,8 @@ export interface VerifyFindingsOptions { skill: SkillDefinition; apiKey?: string; runtime?: RuntimeName; + /** Custom OpenAI-compatible providers to register for the Pi runtime. */ + providers?: ProvidersConfig; model?: string; maxTurns?: number; effort?: Effort; @@ -267,6 +269,7 @@ export async function verifyFindings( tools: options.skill.tools, providerOptions: getRuntimeProviderOptions(runtimeName, { pathToClaudeCodeExecutable: options.pathToClaudeCodeExecutable, + providers: options.providers, }), }); diff --git a/packages/warden/src/skill-builder/agentic.test.ts b/packages/warden/src/skill-builder/agentic.test.ts new file mode 100644 index 00000000..864fe308 --- /dev/null +++ b/packages/warden/src/skill-builder/agentic.test.ts @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; + +vi.mock('../sdk/json-output.js', () => ({ + parseJsonFromOutput: vi.fn(), +})); + +import { parseJsonFromOutput } from '../sdk/json-output.js'; +import { runStructuredSkillBuilderAgent } from './agentic.js'; +import { emptyUsage } from '../sdk/usage.js'; +import type { Runtime } from '../sdk/runtimes/index.js'; +import type { ProvidersConfig } from '../config/schema.js'; + +const parseJsonFromOutputMock = vi.mocked(parseJsonFromOutput); + +const schema = z.object({ ok: z.boolean() }); + +describe('runStructuredSkillBuilderAgent provider forwarding', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('forwards custom providers to the fallback auxiliary JSON repair', async () => { + const providers: ProvidersConfig = { + litellm: { + baseUrl: 'http://localhost:4000/v1', + api: 'openai-completions', + models: [{ id: 'gemma', name: 'gemma' }], + }, + }; + + // Main agent run succeeds with unparseable text; the structured-repair agent + // run then fails (no result), forcing the fallback auxiliary repair path. + const runSkill = vi + .fn() + .mockResolvedValueOnce({ + result: { status: 'success', text: 'NOT JSON', errors: [], usage: emptyUsage() }, + }) + .mockResolvedValue({}); + const runtime = { + name: 'pi', + runSkill, + runAuxiliary: vi.fn(), + runSynthesis: vi.fn(), + } as unknown as Runtime; + + // Primary parse fails; the fallback auxiliary repair succeeds. + parseJsonFromOutputMock + .mockResolvedValueOnce({ success: false, error: 'invalid_json: nope' }) + .mockResolvedValue({ success: true, data: { ok: true }, json: '{"ok":true}', repaired: true }); + + const result = await runStructuredSkillBuilderAgent({ + runtime, + repoPath: '/repo', + skillName: 'security', + systemPrompt: 'sys', + userPrompt: 'user', + schema, + providers, + repair: { apiKey: 'repair-key', model: 'repair-model', maxRetries: 2 }, + }); + + expect(result.data).toEqual({ ok: true }); + // The fallback repair (second parse call) must carry the custom providers. + expect(parseJsonFromOutputMock).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + repair: expect.objectContaining({ providers }), + }), + ); + }); +}); diff --git a/packages/warden/src/skill-builder/agentic.ts b/packages/warden/src/skill-builder/agentic.ts index ad3b9dff..e0f34094 100644 --- a/packages/warden/src/skill-builder/agentic.ts +++ b/packages/warden/src/skill-builder/agentic.ts @@ -1,11 +1,12 @@ import { performance } from 'node:perf_hooks'; import { z } from 'zod'; -import type { ToolName } from '../config/schema.js'; +import type { ProvidersConfig, ToolName } from '../config/schema.js'; import type { UsageStats } from '../types/index.js'; import { parseJsonFromOutput, type ParseJsonFromOutputResult } from '../sdk/json-output.js'; import { buildJsonOutputSection, buildTaggedSection, joinPromptSections } from '../sdk/prompt-sections.js'; import { aggregateUsage, emptyUsage } from '../sdk/usage.js'; import type { Runtime, SkillRunResult } from '../sdk/runtimes/index.js'; +import { getRuntimeProviderOptions } from '../sdk/runtimes/index.js'; const SKILL_BUILDER_READ_TOOLS: ToolName[] = ['Read', 'Grep', 'Glob', 'WebFetch', 'WebSearch']; const SKILL_BUILDER_WRITE_TOOLS: ToolName[] = ['Read', 'Grep', 'Glob', 'Write', 'Edit', 'Bash', 'WebFetch', 'WebSearch']; @@ -135,6 +136,7 @@ async function repairStructuredSkillBuilderOutput(args: { reason: string; model?: string; abortController?: AbortController; + providers?: ProvidersConfig; }): Promise> { const response = await args.runtime.runSkill({ apiKey: args.apiKey, @@ -152,6 +154,7 @@ async function repairStructuredSkillBuilderOutput(args: { maxTurns: STRUCTURED_REPAIR_MAX_TURNS, abortController: args.abortController, }, + providerOptions: getRuntimeProviderOptions(args.runtime.name, { providers: args.providers }), }); if (response.authError) { @@ -208,6 +211,7 @@ export async function runStructuredSkillBuilderAgent(args: { writeAccess?: boolean; abortController?: AbortController; apiKey?: string; + providers?: ProvidersConfig; repair?: { apiKey?: string; model?: string; @@ -229,6 +233,7 @@ export async function runStructuredSkillBuilderAgent(args: { maxTurns: args.maxTurns, abortController: args.abortController, }, + providerOptions: getRuntimeProviderOptions(runtime.name, { providers: args.providers }), }); if (response.authError) { @@ -263,6 +268,7 @@ export async function runStructuredSkillBuilderAgent(args: { reason: parsed.error, model: args.repair?.model ?? args.model, abortController: args.abortController, + providers: args.providers, }); if (skillRepair.usage) { repairUsages.push(skillRepair.usage); @@ -279,6 +285,10 @@ export async function runStructuredSkillBuilderAgent(args: { apiKey: args.repair.apiKey, model: args.repair.model, maxRetries: args.repair.maxRetries, + // Forward custom providers like the primary repair path does, so the + // fallback auxiliary repair stays on the configured (e.g. self-hosted) + // provider instead of the runtime default. + providers: args.providers, }, }); if (auxiliaryRepair.usage) { diff --git a/packages/warden/src/skill-builder/outline.test.ts b/packages/warden/src/skill-builder/outline.test.ts index e7693984..96c8125b 100644 --- a/packages/warden/src/skill-builder/outline.test.ts +++ b/packages/warden/src/skill-builder/outline.test.ts @@ -1,8 +1,11 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, describe, expect, it } from 'vitest'; -import { collectSkillImproveSource } from './outline.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { Runtime, SynthesisRunRequest } from '../sdk/runtimes/index.js'; +import type { ProvidersConfig } from '../config/schema.js'; +import { buildSkillOutline, collectSkillImproveSource } from './outline.js'; +import { SKILL_BUILD_VERSION, SKILL_BUILD_OUTLINE_SCHEMA_VERSION, type SkillBuildOutline } from './outline-contract.js'; import { getBuildStatePath, readSkillBuildState, @@ -161,3 +164,171 @@ prompt: Find security issues. ); }); }); + +describe('buildSkillOutline: custom providers forwarding', () => { + const tempDirs: string[] = []; + + afterEach(() => { + for (const dir of tempDirs) { + rmSync(dir, { recursive: true, force: true }); + } + tempDirs.length = 0; + }); + + it('forwards providerOptions to runSynthesis when providers are configured', async () => { + const rootDir = mkdtempSync(join(tmpdir(), 'warden-outline-providers-')); + tempDirs.push(rootDir); + // Write a minimal warden.yaml so generatedSkillDefinitionSourceFile can load it. + writeFileSync(join(rootDir, 'warden.yaml'), `version: 1\nkind: generated-skill\nname: security\nprompt: Find security issues.\n`, 'utf-8'); + + const providers: ProvidersConfig = { + litellm: { + baseUrl: 'http://localhost:4000', + api: 'openai-completions', + models: [{ id: 'litellm/gpt-4o', name: 'gpt-4o' }], + }, + }; + + // Build a minimal valid outline to return from the stub. + const fakeOutline: SkillBuildOutline = { + version: SKILL_BUILD_OUTLINE_SCHEMA_VERSION, + skill: 'security', + sourceHash: 'placeholder', // overwritten after source is collected + buildVersion: SKILL_BUILD_VERSION, + scopeProfile: { + kind: 'domain', + subject: 'Generic security review', + localContextUsed: false, + observedContext: ['Generic security review'], + unresolvedContext: [], + }, + build: { + phases: [{ id: 'build-tracks', status: 'generated' }], + }, + tracks: [{ + id: 'auth-bypass', + title: 'Authentication bypasses', + goal: 'Find broken authentication checks.', + rationale: 'Authentication bugs are core security issues.', + sourceSignals: ['Auth endpoints'], + owns: ['Missing auth checks'], + excludes: [], + relevanceSignals: ['Session checks'], + evidenceFocus: ['Changed auth conditions'], + checks: ['Trace auth preconditions'], + safeCounterpatterns: ['Explicit user verification'], + falsePositiveTraps: ['Defense-in-depth logging'], + researchHints: [], + }], + }; + + const capturedRequests: SynthesisRunRequest[] = []; + + const mockRuntime: Runtime = { + name: 'pi', + runSkill: vi.fn().mockResolvedValue({ result: undefined }), + runAuxiliary: vi.fn().mockResolvedValue({ success: false, error: 'not used', usage: {} }), + runSynthesis: vi.fn().mockImplementation(async (req: SynthesisRunRequest) => { + capturedRequests.push(req); + // Return a valid outline with the correct sourceHash for the collected source. + const outline = { ...fakeOutline, sourceHash: req.prompt.match(/sourceHash "([^"]+)"/)?.[1] ?? fakeOutline.sourceHash }; + return { success: true as const, data: outline, usage: { inputTokens: 10, outputTokens: 5, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, cacheCreation5mInputTokens: 0, cacheCreation1hInputTokens: 0, webSearchRequests: 0, costUSD: 0 } }; + }), + }; + + await buildSkillOutline({ + skill: { name: 'security', description: 'Security skill', prompt: 'Find security issues.', rootDir }, + runtime: mockRuntime, + providers, + regenerate: true, + }); + + expect(capturedRequests).toHaveLength(1); + const req = capturedRequests[0]!; + // providerOptions must be forwarded: it is a PiProviderOptions object with providers[0].name === 'litellm' + expect(req.providerOptions).toBeDefined(); + const po = req.providerOptions as { providers: { name: string }[] }; + expect(po.providers[0]!.name).toBe('litellm'); + }); + + it('forwards providerOptions through the repoPath agentic branch', async () => { + const rootDir = mkdtempSync(join(tmpdir(), 'warden-outline-providers-repo-')); + tempDirs.push(rootDir); + writeFileSync(join(rootDir, 'warden.yaml'), `version: 1\nkind: generated-skill\nname: security\nprompt: Find security issues.\n`, 'utf-8'); + + const providers: ProvidersConfig = { + litellm: { + baseUrl: 'http://localhost:4000', + api: 'openai-completions', + models: [{ id: 'litellm/gpt-4o', name: 'gpt-4o' }], + }, + }; + + const fakeOutline: SkillBuildOutline = { + version: SKILL_BUILD_OUTLINE_SCHEMA_VERSION, + skill: 'security', + sourceHash: 'placeholder', + buildVersion: SKILL_BUILD_VERSION, + scopeProfile: { + kind: 'domain', + subject: 'Generic security review', + localContextUsed: false, + observedContext: ['Generic security review'], + unresolvedContext: [], + }, + build: { + phases: [{ id: 'build-tracks', status: 'generated' }], + }, + tracks: [{ + id: 'auth-bypass', + title: 'Authentication bypasses', + goal: 'Find broken authentication checks.', + rationale: 'Authentication bugs are core security issues.', + sourceSignals: ['Auth endpoints'], + owns: ['Missing auth checks'], + excludes: [], + relevanceSignals: ['Session checks'], + evidenceFocus: ['Changed auth conditions'], + checks: ['Trace auth preconditions'], + safeCounterpatterns: ['Explicit user verification'], + falsePositiveTraps: ['Defense-in-depth logging'], + researchHints: [], + }], + }; + + // The agentic branch (repoPath set) runs through runtime.runSkill and parses + // the structured outline from result.text. Echo the correct sourceHash back so + // validateOutlineIdentity passes. + const capturedRequests: { providerOptions?: unknown }[] = []; + const mockRuntime: Runtime = { + name: 'pi', + runSkill: vi.fn().mockImplementation(async (req: { userPrompt: string; providerOptions?: unknown }) => { + capturedRequests.push({ providerOptions: req.providerOptions }); + const sourceHash = req.userPrompt.match(/sourceHash "([^"]+)"/)?.[1] ?? fakeOutline.sourceHash; + return { + result: { + status: 'success' as const, + text: JSON.stringify({ ...fakeOutline, sourceHash }), + errors: [], + usage: { inputTokens: 10, outputTokens: 5, costUSD: 0 }, + }, + }; + }), + runAuxiliary: vi.fn().mockResolvedValue({ success: false, error: 'not used', usage: {} }), + runSynthesis: vi.fn().mockResolvedValue({ success: false, error: 'should not be called', usage: {} }), + }; + + await buildSkillOutline({ + skill: { name: 'security', description: 'Security skill', prompt: 'Find security issues.', rootDir }, + runtime: mockRuntime, + repoPath: rootDir, + providers, + regenerate: true, + }); + + expect(mockRuntime.runSynthesis).not.toHaveBeenCalled(); + expect(capturedRequests).toHaveLength(1); + const po = capturedRequests[0]!.providerOptions as { providers: { name: string }[] }; + expect(po.providers[0]!.name).toBe('litellm'); + }); +}); diff --git a/packages/warden/src/skill-builder/outline.ts b/packages/warden/src/skill-builder/outline.ts index b207f6cb..6fa5975a 100644 --- a/packages/warden/src/skill-builder/outline.ts +++ b/packages/warden/src/skill-builder/outline.ts @@ -1,9 +1,9 @@ import { createHash } from 'node:crypto'; import { existsSync } from 'node:fs'; import { basename } from 'node:path'; -import type { SkillDefinition } from '../config/schema.js'; +import type { ProvidersConfig, SkillDefinition } from '../config/schema.js'; import type { Runtime, RuntimeName } from '../sdk/runtimes/index.js'; -import { getRuntime } from '../sdk/runtimes/index.js'; +import { getRuntime, getRuntimeProviderOptions } from '../sdk/runtimes/index.js'; import { runStructuredSkillBuilderAgent, StructuredSkillBuilderAgentError } from './agentic.js'; import { GENERATED_SKILL_DEFINITION_FILE, @@ -51,6 +51,7 @@ export interface BuildSkillOutlineOptions { repairMaxRetries?: number; onStatus?: (message: string) => void; source?: SkillBuildSource; + providers?: ProvidersConfig; } export class SkillBuildOutlineError extends Error { @@ -359,6 +360,7 @@ export async function buildSkillOutline( maxTurns: options.maxTurns ?? SKILL_BUILD_MAX_TURNS, apiKey, abortController: options.abortController, + providers: options.providers, repair: { apiKey, model: options.repairModel, @@ -408,6 +410,7 @@ export async function buildSkillOutline( maxTokens: SKILL_BUILD_MAX_TOKENS, timeout: SKILL_BUILD_TIMEOUT_MS, maxRetries, + providerOptions: getRuntimeProviderOptions(runtime.name, { providers: options.providers }), }); if (!result.success) { diff --git a/packages/warden/src/skill-builder/skill.ts b/packages/warden/src/skill-builder/skill.ts index 38f47147..9e8e75ab 100644 --- a/packages/warden/src/skill-builder/skill.ts +++ b/packages/warden/src/skill-builder/skill.ts @@ -3,6 +3,7 @@ import { existsSync } from 'node:fs'; import { performance } from 'node:perf_hooks'; import { parse as parseYaml } from 'yaml'; import { aggregateUsage } from '../sdk/usage.js'; +import type { ProvidersConfig } from '../config/schema.js'; import type { Runtime } from '../sdk/runtimes/index.js'; import { runStructuredSkillBuilderAgent, StructuredSkillBuilderAgentError } from './agentic.js'; import type { UsageStats } from '../types/index.js'; @@ -462,6 +463,7 @@ export async function buildGeneratedSkill(args: { repairMaxRetries?: number; authoringSkillRoot?: string; onStatus?: (message: string) => void; + providers?: ProvidersConfig; }): Promise { const startedAt = performance.now(); const statePath = getBuildStatePath(args.rootDir); @@ -522,6 +524,7 @@ export async function buildGeneratedSkill(args: { maxTurns, apiKey: args.apiKey, abortController: args.abortController, + providers: args.providers, repair, }); @@ -551,6 +554,7 @@ export async function buildGeneratedSkill(args: { apiKey: args.apiKey, writeAccess: true, abortController: args.abortController, + providers: args.providers, repair, }); @@ -598,6 +602,7 @@ export async function buildGeneratedSkill(args: { maxTurns: Math.min(maxTurns, defaultValidationMaxTurns()), apiKey: args.apiKey, abortController: args.abortController, + providers: args.providers, repair, }); reviewResults.push(review); @@ -637,6 +642,7 @@ export async function buildGeneratedSkill(args: { apiKey: args.apiKey, writeAccess: true, abortController: args.abortController, + providers: args.providers, repair, }); revisionResults.push(revision);