From 44ec9b0c2196448005eb68d9d05cf103a8e60c26 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Wed, 15 Jul 2026 11:22:40 -0400 Subject: [PATCH 1/8] =?UTF-8?q?fix(pi):=20review=20fixes=20=E2=80=94=20typ?= =?UTF-8?q?ed=20effort,=20live=20model=20fallback,=20menu-declared=20varia?= =?UTF-8?q?nts,=20shared=20session=20machinery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from the review of #853: - effort is a ThinkingLevel end to end, validated at the parse boundary: a typo'd effort_pi in remote frontmatter is logged and dropped (the model-table default applies) instead of riding into a session through a cast. promptModelFor/taskModelSpec take the Harness enum, not a string. - taskModelSpec no longer bakes in DEFAULT_TASK_MODEL, which made the call sites' `?? pick.model` dead code: a pi task whose prompt names no model_pi now degrades to the switchboard's pi model, not sonnet-over-pi. Each harness falls back within its own column. - terra reverts to low in MODEL_CAPABILITIES: the table tunes the linear run only, and orchestrator agents already carry their own effort via prompt frontmatter (which overrides the table), so the medium row was a side effect on linear runs that the orchestrator never needed. - FRAMEWORK_VARIANT_ALIASES and the startsWith heuristic are gone: skill-menu.json entries now declare `group`/`framework`/`default` (context-mill PR on experiment/orchestrator), so variant resolution is an exact lookup, misses preflight before any agent runs (log + a new `orchestrator skill variant missing` capture), and the variant test pins the real menu contract instead of a fixture that mirrors the code. - new pi/shared.ts holds the session machinery both entry points use — gateway/registry wiring, hermetic loader, coding-tool factories, the session event watcher, error classification, usage capture — so run() and runPiTask() are configurations of one implementation and the 429 sniff / analytics shape can't drift. The linear run gains the parseable usage log line; the per-task line drops its modelId-as-task fallback. - loadAgentRegistry passes the menu entry's flow into parseAgentPrompt as the frontmatter fallback, so a prompt the menu placed in a flow is never silently dropped by the registry filter. Generated-By: PostHog Code Task-Id: fafc230d-6f14-4e4d-9462-0e7f18a1eec1 --- .../__tests__/agent-prompt-loader.test.ts | 62 ++- src/lib/agent/agent-prompt-loader.ts | 63 +-- .../runner/__tests__/switchboard.test.ts | 15 +- .../harness/pi/__tests__/env-lockdown.test.ts | 2 +- .../harness/pi/__tests__/status-line.test.ts | 2 +- src/lib/agent/runner/harness/pi/gateway.ts | 9 +- src/lib/agent/runner/harness/pi/index.ts | 376 ++++------------ src/lib/agent/runner/harness/pi/shared.ts | 408 ++++++++++++++++++ src/lib/agent/runner/harness/pi/subagent.ts | 16 +- src/lib/agent/runner/harness/pi/task.ts | 222 +++------- src/lib/agent/runner/harness/types.ts | 3 +- .../__tests__/variant-resolution.test.ts | 176 +++++--- .../orchestrator/orchestrator-runner.ts | 92 ++-- src/lib/agent/runner/switchboard/models.ts | 30 +- src/lib/wizard-tools.ts | 12 +- 15 files changed, 842 insertions(+), 646 deletions(-) create mode 100644 src/lib/agent/runner/harness/pi/shared.ts diff --git a/src/lib/agent/__tests__/agent-prompt-loader.test.ts b/src/lib/agent/__tests__/agent-prompt-loader.test.ts index 8b7d5bb05..b25ed90bb 100644 --- a/src/lib/agent/__tests__/agent-prompt-loader.test.ts +++ b/src/lib/agent/__tests__/agent-prompt-loader.test.ts @@ -15,6 +15,7 @@ import { } from '../agent-prompt-loader'; import { QueueStore } from '@lib/agent/runner/sequence/orchestrator/queue'; import { HostResolution } from '@lib/host-resolution'; +import { Harness } from '@lib/constants'; function tmpDir(): string { return fs.mkdtempSync(path.join(os.tmpdir(), 'agent-loader-test-')); @@ -57,16 +58,40 @@ Add at least one capture call. it('resolves the per-harness model + effort, not 1:1 across providers', () => { const p = parseAgentPrompt(sample, 'fallback'); - expect(promptModelFor(p, 'pi')).toEqual({ + expect(promptModelFor(p, Harness.pi)).toEqual({ model: 'openai/gpt-5.6-terra', effort: 'medium', }); - expect(promptModelFor(p, 'anthropic')).toEqual({ + expect(promptModelFor(p, Harness.anthropic)).toEqual({ model: 'claude-sonnet-4-6', effort: undefined, }); }); + it('drops an effort that is not a ThinkingLevel — remote typos never reach a session', () => { + const p = parseAgentPrompt( + '---\nmodel_pi: m\neffort_pi: mediun\neffort_sdk: high\n---\nx', + 'capture', + ); + expect(p.effortPi).toBeUndefined(); + expect(p.effortSdk).toBe('high'); + }); + + it('falls back to the menu entry flow when frontmatter omits it', () => { + const p = parseAgentPrompt( + '---\ntype: install\n---\nx', + 'install', + 'my-flow', + ); + expect(p.flow).toBe('my-flow'); + const declared = parseAgentPrompt( + '---\nflow: audit\n---\nx', + 'install', + 'my-flow', + ); + expect(declared.flow).toBe('audit'); + }); + it('strips inline comments and keeps the body', () => { const p = parseAgentPrompt(sample, 'fallback'); expect(p.modelPi).not.toContain('#'); @@ -93,9 +118,10 @@ Add at least one capture call. ); }); - it('defaults missing array fields to empty and model to undefined', () => { + it('defaults missing array fields to empty and models to undefined', () => { const p = parseAgentPrompt('no frontmatter at all', 'stub'); - expect(p.model).toBeUndefined(); + expect(p.modelPi).toBeUndefined(); + expect(p.modelSdk).toBeUndefined(); expect(p.skills).toEqual([]); expect(p.dependsOn).toEqual([]); expect(p.body).toBe('no frontmatter at all'); @@ -208,11 +234,11 @@ describe('resolveTask', () => { it('resolves per-harness model + effort from the prompt', () => { const registry = registryOf([prompt]); const task = store.enqueue({ type: 'capture' }); - expect(taskModelSpec(registry, task, 'pi')).toEqual({ + expect(taskModelSpec(registry, task, Harness.pi)).toEqual({ model: 'openai/gpt-5.6-luna', effort: 'low', }); - expect(taskModelSpec(registry, task, 'anthropic').model).toBe( + expect(taskModelSpec(registry, task, Harness.anthropic).model).toBe( 'claude-haiku-4-5-20251001', ); }); @@ -220,7 +246,7 @@ describe('resolveTask', () => { it('prefers the enqueue model override over the prompt model', () => { const registry = registryOf([prompt]); const task = store.enqueue({ type: 'capture', model: 'override-x' }); - expect(taskModelSpec(registry, task, 'pi').model).toBe('override-x'); + expect(taskModelSpec(registry, task, Harness.pi).model).toBe('override-x'); }); it("appends upstream dependencies' handoffs as context", () => { @@ -296,19 +322,27 @@ describe('taskModelSpec', () => { 'capture', ); - it('prefers the enqueue override, then the prompt, then the default', () => { + it('prefers the enqueue override, then the prompt; no default baked in', () => { const registry = registryOf([prompt]); const task = { type: 'capture' }; expect( - taskModelSpec(registry, { ...task, model: 'override' } as never, 'pi') - .model, + taskModelSpec( + registry, + { ...task, model: 'override' } as never, + Harness.pi, + ).model, ).toBe('override'); - expect(taskModelSpec(registry, task as never, 'pi').model).toBe( + expect(taskModelSpec(registry, task as never, Harness.pi).model).toBe( 'prompt-model', ); - expect(taskModelSpec(registryOf([]), task as never, 'pi').model).toBe( - 'claude-sonnet-4-6', - ); + // An empty column stays undefined — the CALLER falls back to its + // switchboard pick, so a pi run degrades to the pi model, not sonnet. + expect( + taskModelSpec(registry, task as never, Harness.anthropic).model, + ).toBeUndefined(); + expect( + taskModelSpec(registryOf([]), task as never, Harness.pi).model, + ).toBeUndefined(); }); }); diff --git a/src/lib/agent/agent-prompt-loader.ts b/src/lib/agent/agent-prompt-loader.ts index 65123deab..b6885fcb3 100644 --- a/src/lib/agent/agent-prompt-loader.ts +++ b/src/lib/agent/agent-prompt-loader.ts @@ -21,7 +21,12 @@ import type { } from './runner/sequence/orchestrator/queue'; import type { ResolvedTask } from './runner/sequence/orchestrator/executor'; import type { HostResolution } from '@lib/host-resolution'; -import { DEFAULT_AGENT_MODEL } from '@lib/constants'; +import { Harness } from '@lib/constants'; +import { + isThinkingLevel, + type ThinkingLevel, +} from './runner/switchboard/models'; +import { logToFile } from '@utils/debug'; /** * The basics the client injects around every agent-prompt body. The `/agents/` @@ -101,9 +106,6 @@ export function assembleSeedPrompt( return [projectContext(ctx), SEED_BASICS, body].join('\n\n'); } -/** Used when neither the enqueue call nor the prompt frontmatter names a model. */ -const DEFAULT_TASK_MODEL = DEFAULT_AGENT_MODEL; - /** Orchestrator tools are MCP tools under the `posthog-wizard` server. Frontmatter * names them short (e.g. `enqueue_task`); the SDK gates on the full name. */ const ORCHESTRATOR_TOOL_PREFIX = 'mcp__posthog-wizard__'; @@ -125,9 +127,9 @@ export interface AgentPrompt { /** Per-profile model + effort. `pi` = the gpt/pi harness, `sdk` = the anthropic * harness. The mapping is not 1:1 across providers, so each agent names both. */ modelPi?: string; - effortPi?: string; + effortPi?: ThinkingLevel; modelSdk?: string; - effortSdk?: string; + effortSdk?: ThinkingLevel; skills: string[]; allowedTools: string[]; disallowedTools: string[]; @@ -139,9 +141,9 @@ export interface AgentPrompt { * column, anything else the sdk (anthropic) column. */ export function promptModelFor( prompt: AgentPrompt, - harness: string, -): { model?: string; effort?: string } { - const pi = harness === 'pi'; + harness: Harness, +): { model?: string; effort?: ThinkingLevel } { + const pi = harness === Harness.pi; return { model: pi ? prompt.modelPi : prompt.modelSdk, effort: pi ? prompt.effortPi : prompt.effortSdk, @@ -210,11 +212,14 @@ function toStringArray(value: unknown): string[] { * frontmatter is a small, known schema (scalars and inline `[a, b]` arrays), so * a tiny parser covers it without a YAML dependency. Inline `# comments` after a * value are stripped. `fallbackType` is the menu id, used when the body omits - * `type:`. + * `type:`; `fallbackFlow` is the menu entry's flow, used when the body omits + * `flow:` — so a prompt the menu placed in a flow is never silently dropped + * by the registry's flow filter. */ export function parseAgentPrompt( text: string, fallbackType: string, + fallbackFlow?: string, ): AgentPrompt { const match = text.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/); const frontmatter = match ? match[1] : ''; @@ -239,15 +244,25 @@ export function parseAgentPrompt( } const str = (v: unknown) => (typeof v === 'string' ? v : undefined); + // Effort is remote data; a typo must not ride into a session as a bogus + // reasoning level. Validate here so downstream code carries ThinkingLevel. + const effort = (v: unknown, key: string): ThinkingLevel | undefined => { + if (v === undefined) return undefined; + if (isThinkingLevel(v)) return v; + logToFile( + `[agent-prompt] ${fallbackType}: ignoring invalid ${key} "${String(v)}"`, + ); + return undefined; + }; return { type: typeof fields.type === 'string' ? fields.type : fallbackType, label: typeof fields.label === 'string' ? fields.label : undefined, - flow: typeof fields.flow === 'string' ? fields.flow : undefined, + flow: typeof fields.flow === 'string' ? fields.flow : fallbackFlow, seed: fields.seed === 'true', modelPi: str(fields.model_pi), - effortPi: str(fields.effort_pi), + effortPi: effort(fields.effort_pi, 'effort_pi'), modelSdk: str(fields.model_sdk), - effortSdk: str(fields.effort_sdk), + effortSdk: effort(fields.effort_sdk, 'effort_sdk'), skills: toStringArray(fields.skills), allowedTools: toStringArray(fields.allowedTools), disallowedTools: toStringArray(fields.disallowedTools), @@ -286,7 +301,7 @@ export async function loadAgentRegistry( const prompts = await Promise.all( entries.map(async (entry) => { const text = await fetchText(entry.downloadUrl); - return parseAgentPrompt(text, entry.id); + return parseAgentPrompt(text, entry.id, entry.flow); }), ); @@ -386,20 +401,20 @@ export function resolveTask( } /** The model + effort a task runs on for a harness: enqueue override, then the - * prompt's per-profile frontmatter, then the default model. */ + * prompt's per-profile frontmatter. No default is baked in here — the caller + * falls back to its switchboard pick, so each harness degrades to its own + * column's model rather than everything collapsing to the anthropic default. */ export function taskModelSpec( registry: AgentRegistry, task: QueuedTask, - harness: string, -): { model: string; effort?: string } { - const picked = promptModelFor( - registry.get(task.type) ?? EMPTY_PROMPT, - harness, - ); + harness: Harness, +): { model?: string; effort?: ThinkingLevel } { + const prompt = registry.get(task.type); + const picked = prompt + ? promptModelFor(prompt, harness) + : { model: undefined, effort: undefined }; return { - model: task.model ?? picked.model ?? DEFAULT_TASK_MODEL, + model: task.model ?? picked.model, effort: picked.effort, }; } - -const EMPTY_PROMPT = {} as AgentPrompt; diff --git a/src/lib/agent/runner/__tests__/switchboard.test.ts b/src/lib/agent/runner/__tests__/switchboard.test.ts index 41051092f..82d6fa7c6 100644 --- a/src/lib/agent/runner/__tests__/switchboard.test.ts +++ b/src/lib/agent/runner/__tests__/switchboard.test.ts @@ -328,6 +328,8 @@ describe('switchboard modelCapabilities', () => { expect(modelCapabilities(GPT5_MODEL).thinkingLevel).toBe('low'); expect(modelCapabilities(GPT5_MINI_MODEL).thinkingLevel).toBe('medium'); // The gpt-5.6 line + gpt-5.5 are reasoning models despite the openai/ prefix; they opt in past the default-off. + // All low in the table — the table tunes the LINEAR run; orchestrator + // agents raise effort per task via their prompt frontmatter instead. for (const m of [ GPT5_6_LUNA_MODEL, GPT5_6_TERRA_MODEL, @@ -335,12 +337,8 @@ describe('switchboard modelCapabilities', () => { GPT5_5_MODEL, ]) { expect(modelCapabilities(m).reasoning).toBe(true); + expect(modelCapabilities(m).thinkingLevel).toBe('low'); } - // luna/sol/5.5 stay low (fast); terra runs medium as the sonnet-tier parallel. - expect(modelCapabilities(GPT5_6_LUNA_MODEL).thinkingLevel).toBe('low'); - expect(modelCapabilities(GPT5_6_TERRA_MODEL).thinkingLevel).toBe('medium'); - expect(modelCapabilities(GPT5_6_SOL_MODEL).thinkingLevel).toBe('low'); - expect(modelCapabilities(GPT5_5_MODEL).thinkingLevel).toBe('low'); // Anthropic default carries no explicit effort — the harness default stands. expect( modelCapabilities(DEFAULT_AGENT_MODEL).thinkingLevel, @@ -406,15 +404,16 @@ describe('switchboard wizard-pi-effort flag', () => { }); it('opts out with applyEffortFlag:false — orchestrator tasks keep the table effort', () => { - // The flag is a linear-run knob; a per-task agent ignores it and keeps its - // own tuned level (terra medium), even with the flag set to high. + // The flag is a linear-run knob; a per-task agent ignores it and keeps the + // table level (its own effort comes from prompt frontmatter instead), + // even with the flag set to high. expect( modelCapabilities( GPT5_6_TERRA_MODEL, { ...PI_ON, [WIZARD_PI_EFFORT_FLAG_KEY]: 'high' }, { applyEffortFlag: false }, ).thinkingLevel, - ).toBe('medium'); + ).toBe('low'); expect( modelCapabilities( GPT5_6_LUNA_MODEL, diff --git a/src/lib/agent/runner/harness/pi/__tests__/env-lockdown.test.ts b/src/lib/agent/runner/harness/pi/__tests__/env-lockdown.test.ts index 1034e0bca..e6600e234 100644 --- a/src/lib/agent/runner/harness/pi/__tests__/env-lockdown.test.ts +++ b/src/lib/agent/runner/harness/pi/__tests__/env-lockdown.test.ts @@ -4,7 +4,7 @@ * drops everything else — the leak that exposed the test key before. */ -import { buildScrubbedEnv } from '..'; +import { buildScrubbedEnv } from '../shared'; describe('buildScrubbedEnv', () => { const saved = { ...process.env }; diff --git a/src/lib/agent/runner/harness/pi/__tests__/status-line.test.ts b/src/lib/agent/runner/harness/pi/__tests__/status-line.test.ts index b2920ab18..5f92a763a 100644 --- a/src/lib/agent/runner/harness/pi/__tests__/status-line.test.ts +++ b/src/lib/agent/runner/harness/pi/__tests__/status-line.test.ts @@ -6,7 +6,7 @@ import { describe, it, expect } from 'vitest'; import { AgentSignals } from '@lib/agent/signals'; -import { lastStatusLine } from '..'; +import { lastStatusLine } from '../shared'; const S = AgentSignals.STATUS; // '[STATUS]' diff --git a/src/lib/agent/runner/harness/pi/gateway.ts b/src/lib/agent/runner/harness/pi/gateway.ts index 4328d83f7..5a90e5334 100644 --- a/src/lib/agent/runner/harness/pi/gateway.ts +++ b/src/lib/agent/runner/harness/pi/gateway.ts @@ -70,9 +70,10 @@ export interface GatewayProviderInputs { // Linear runs honour the wizard-pi-effort flag; orchestrator tasks pass false // so each per-agent model keeps its own tuned effort from the table. applyEffortFlag?: boolean; - // Explicit per-agent effort from the prompt frontmatter — overrides the table - // default for a reasoning model when set. - effort?: string; + // Explicit per-agent effort from the prompt frontmatter — validated to a + // ThinkingLevel at the parse boundary; overrides the table default for a + // reasoning model when set. + effort?: ThinkingLevel; } /** @@ -103,7 +104,7 @@ export function buildGatewayProvider(inputs: GatewayProviderInputs): { // An explicit frontmatter effort wins over the table for a reasoning model. const caps = effort && tableCaps.reasoning - ? { ...tableCaps, thinkingLevel: effort as ThinkingLevel } + ? { ...tableCaps, thinkingLevel: effort } : tableCaps; const baseUrl = api === 'openai-completions' ? `${gatewayUrl}/v1` : gatewayUrl; diff --git a/src/lib/agent/runner/harness/pi/index.ts b/src/lib/agent/runner/harness/pi/index.ts index 16d815f27..481fd4a87 100644 --- a/src/lib/agent/runner/harness/pi/index.ts +++ b/src/lib/agent/runner/harness/pi/index.ts @@ -16,17 +16,22 @@ import fs from 'fs'; import path from 'path'; import { getUI } from '@ui'; import { getLogFilePath, logToFile } from '@utils/debug'; -import { - Harness, - WIZARD_REMARK_EVENT_NAME, - WIZARD_USER_AGENT, -} from '@lib/constants'; +import { Harness, WIZARD_REMARK_EVENT_NAME } from '@lib/constants'; import { analytics } from '@utils/analytics'; import { AgentErrorType } from '@lib/agent/agent-interface'; -import { AgentSignals, REMARK_INSTRUCTION } from '@lib/agent/signals'; +import { REMARK_INSTRUCTION } from '@lib/agent/signals'; import { AgentOutputSignals } from '@lib/agent/output-signals'; import { getWizardCommandments } from '@lib/agent/commandments'; -import { buildGatewayProvider, GATEWAY_PROVIDER } from './gateway'; +import { + captureAgentUsage, + classifyRunError, + connectPostHogMcp, + createHermeticSession, + piCodingToolFactories, + resolveGatewayModel, + startRunClock, + watchSession, +} from './shared'; import type { AgentResult, AgentHarness, @@ -94,113 +99,6 @@ function piMcpContext(boot: BootstrapResult, instructions?: string): string { ].join('\n'); } -/** - * The ONLY environment variables pi's tool subprocesses (bash → npm/pip/…) are - * allowed to see. Everything else — every secret (POSTHOG_PERSONAL_API_KEY, - * ANTHROPIC_*, AWS_*), every ambient credential, the parent process's whole env - * — is dropped before a child is spawned. pi's own gateway auth is programmatic - * (the access token never lives in env), so a minimal env costs the agent - * nothing while closing the leak that exposed the key before. Kept to what a - * package manager genuinely needs to run. - */ -const ALLOWED_SUBPROCESS_ENV_KEYS = [ - 'PATH', - 'HOME', - 'SHELL', - 'USER', - 'LOGNAME', - 'TMPDIR', - 'TMP', - 'TEMP', - 'TERM', - 'LANG', - 'LC_ALL', - 'LC_CTYPE', - 'NODE_EXTRA_CA_CERTS', - 'SSL_CERT_FILE', - 'SSL_CERT_DIR', - 'HTTP_PROXY', - 'HTTPS_PROXY', - 'NO_PROXY', - 'http_proxy', - 'https_proxy', - 'no_proxy', -]; - -/** A fresh subprocess env holding only the allowlisted keys present in process.env. */ -export function buildScrubbedEnv(): NodeJS.ProcessEnv { - const env: NodeJS.ProcessEnv = {}; - for (const key of ALLOWED_SUBPROCESS_ENV_KEYS) { - const value = process.env[key]; - if (value !== undefined) env[key] = value; - } - return env; -} - -/** - * Tag a tool with an execution mode (mutates + returns it). Read-only tools are - * `parallel` so a single turn that batches independent reads/searches runs them - * at once; mutating/install tools are `sequential` so a batch never races writes - * or concurrent installs. pi-agent-core runs a batch in parallel only when no - * tool in it is `sequential`. - */ -export function withMode(tool: T, mode: 'sequential' | 'parallel'): T { - (tool as { executionMode?: 'sequential' | 'parallel' }).executionMode = mode; - return tool; -} - -/** Pull plain text out of a pi AgentMessage (content is text/image blocks). */ -export function extractText(message: unknown): string { - const content = (message as { content?: unknown })?.content; - if (typeof content === 'string') return content; - if (Array.isArray(content)) { - return content - .filter((c): c is { type: string; text: string } => { - const block = c as { type?: string; text?: unknown }; - return block?.type === 'text' && typeof block.text === 'string'; - }) - .map((c) => c.text) - .join(''); - } - return ''; -} - -/** - * Surface `[DASHBOARD_URL]` / `[NOTEBOOK_URL]` markers the agent prints (after - * the MCP creates them) into the outro link, mirroring the anthropic path's - * signal parsing (#9). The marker carries the URL the MCP returned. - */ -export function applyOutroMarkers(textBlock: string): void { - const markers: Array<[string, (url: string) => void]> = [ - [AgentSignals.DASHBOARD_URL, (url) => getUI().setDashboardUrl(url)], - [AgentSignals.NOTEBOOK_URL, (url) => getUI().setNotebookUrl(url)], - ]; - for (const [marker, apply] of markers) { - const idx = textBlock.indexOf(marker); - if (idx === -1) continue; - const url = textBlock - .slice(idx + marker.length) - .trim() - .split(/\s/)[0]; - if (url) apply(url); - } -} - -/** - * The text of the last `[STATUS] …` line in a block, if any. Last wins so the - * spinner shows the most recent action when a turn prints several. - */ -export function lastStatusLine(textBlock: string): string | undefined { - let status: string | undefined; - for (const line of textBlock.split('\n')) { - const idx = line.indexOf(AgentSignals.STATUS); - if (idx !== -1) { - status = line.slice(idx + AgentSignals.STATUS.length).trim(); - } - } - return status || undefined; -} - /** Cap on completion-guard re-prompts while tasks remain open (see the run loop). */ const MAX_CONTINUE_NUDGES = 20; @@ -229,19 +127,8 @@ export const piBackend: AgentHarness = { spinner.start(config.spinnerMessage ?? 'Customizing your PostHog setup...'); // Same `agent completed`/`agent aborted` shape as anthropic. - const startTime = Date.now(); const signals = new AgentOutputSignals(); - let assistantTurns = 0; - // Tool calls across the whole run. Zero means the agent only ever produced - // text and never acted — a no-op that leaves the project untouched. - let toolCalls = 0; - const runDurations = () => { - const durationMs = Date.now() - startTime; - return { - duration_ms: durationMs, - duration_seconds: Math.round(durationMs / 1000), - }; - }; + const runDurations = startRunClock(); const captureAborted = () => analytics.wizardCapture('agent aborted', { ...runDurations(), @@ -249,41 +136,20 @@ export const piBackend: AgentHarness = { }); try { - const { - createAgentSession, - DefaultResourceLoader, - SessionManager, - AuthStorage, - ModelRegistry, - getAgentDir, - createLsToolDefinition, - createFindToolDefinition, - createGrepToolDefinition, - createBashToolDefinition, - createReadToolDefinition, - createEditToolDefinition, - createWriteToolDefinition, - } = await import('@earendil-works/pi-coding-agent'); - - // the claude-agent-sdk path. The provider spec is shared with the - // orchestrator's per-task sessions (gateway.ts). - const { provider, caps, gatewayUrl } = buildGatewayProvider({ - gatewayUrl: boot.credentials.host.gatewayUrl, - accessToken: boot.credentials.accessToken, - wizardMetadata: boot.wizardMetadata, - wizardFlags: boot.wizardFlags, - modelId, - }); - const registry = ModelRegistry.inMemory(AuthStorage.create()); - registry.registerProvider(GATEWAY_PROVIDER, provider as never); - - const model = registry.find(GATEWAY_PROVIDER, modelId); - if (!model) { + const sdk = await import('@earendil-works/pi-coding-agent'); + const { getAgentDir } = sdk; + + // The gateway provider spec + registry wiring is shared with the + // orchestrator's per-task sessions (shared.ts / gateway.ts). Linear runs + // honour the wizard-pi-effort flag (the default). + const gateway = resolveGatewayModel(sdk, boot, modelId); + if (!gateway) { return { error: AgentErrorType.API_ERROR, message: 'pi: gateway model could not be resolved', }; } + const { registry, model, caps, gatewayUrl } = gateway; // System prompt = wizard commandments. Skip project context files / // user extensions / skills so the run is hermetic; skills discovery is a @@ -314,48 +180,16 @@ export const piBackend: AgentHarness = { const { prewarmYaraScanner } = await import('@lib/yara-hooks'); void prewarmYaraScanner(); - // Wire the real PostHog MCP into pi (#10): load pi's MCP adapter and point - // it at the hosted MCP the anthropic path uses, so dashboards/insights are - // created through the sanctioned MCP. Best-effort — if it can't load or - // connect, the run continues (minus the dashboard step) rather than failing - // the whole integration. The security factory is always first. + // Wire the real PostHog MCP into pi (#10), best-effort — if it can't + // load or connect, the run continues (minus the dashboard step) rather + // than failing the whole integration. The security factory is always + // first. const extensionFactories = [security.factory] as Array< (pi: unknown) => void >; - let mcpCleanup: (() => void) | undefined; - let mcpInstructions: string | undefined; - try { - const { setupPostHogMcp } = await import('./mcp'); - const mcp = await setupPostHogMcp({ - agentDir: getAgentDir(), - mcpUrl: boot.credentials.host.mcpUrl, - accessToken: boot.credentials.accessToken, - userAgent: WIZARD_USER_AGENT, - }); - extensionFactories.push(mcp.extensionFactory); - mcpCleanup = mcp.cleanup; - mcpInstructions = mcp.instructions; - } catch (err) { - logToFile(`[pi] PostHog MCP setup skipped: ${String(err)}`); - } - - const resourceLoader = new DefaultResourceLoader({ - cwd: session.installDir, - agentDir: getAgentDir(), - systemPrompt: - getWizardCommandments() + - '\n' + - PI_RUNTIME_NOTES + - '\n' + - piMcpContext(boot, mcpInstructions), - noExtensions: true, - noSkills: true, - noContextFiles: true, - noPromptTemplates: true, - noThemes: true, - extensionFactories, - }); - await resourceLoader.reload(); + const mcp = await connectPostHogMcp(sdk, boot, '[pi]'); + if (mcp.extensionFactory) extensionFactories.push(mcp.extensionFactory); + const mcpCleanup = mcp.cleanup; // Wizard capabilities as custom tools (pi has no MCP): skill // discovery/install + fenced .env edits, same names as the MCP server so @@ -368,31 +202,24 @@ export const piBackend: AgentHarness = { const { createDispatchAgentTool } = await import('./subagent'); // Created once so the run loop can read the store for the completion guard. const wizardTaskTools = createWizardPiTaskTools(); - // The one bash the agent (and its subagents) may use: every subprocess it - // spawns gets a scrubbed env, so no secret or ambient variable reaches an - // `npm install`. Shared with the subagent so the lockdown is inherited. - const scrubbedBash = withMode( - createBashToolDefinition(session.installDir, { - spawnHook: (ctx) => ({ ...ctx, env: buildScrubbedEnv() }), - }), - 'sequential', - ); + // Built-ins re-registered explicitly (`noTools: 'builtin'` in the shared + // session builder disables pi's defaults): reads/searches parallel, + // edit/write/bash sequential, bash on the scrubbed env. The one bash is + // shared with the subagent so the lockdown is inherited. + const coding = piCodingToolFactories(sdk, session.installDir); + const scrubbedBash = coding.bash(); const customTools = [ - // Built-ins re-registered explicitly. `noTools: 'builtin'` disables pi's - // defaults so we can supply the env-scrubbed bash above; read/edit/write - // are the stock definitions. Reads run in parallel so a batched turn of - // independent reads executes at once; edit/write/bash stay sequential. - withMode(createReadToolDefinition(session.installDir), 'parallel'), - withMode(createEditToolDefinition(session.installDir), 'sequential'), - withMode(createWriteToolDefinition(session.installDir), 'sequential'), + coding.read(), + coding.edit(), + coding.write(), scrubbedBash, // Native ls/find/grep so the agent explores with proper tools instead // of fence-blocked `bash {ls/find}` (the profiled retry-spirals came // from this gap). Parallel — exploration batches cleanly. - withMode(createLsToolDefinition(session.installDir), 'parallel'), - withMode(createFindToolDefinition(session.installDir), 'parallel'), - withMode(createGrepToolDefinition(session.installDir), 'parallel'), + coding.ls(), + coding.find(), + coding.grep(), ...createWizardPiTools({ workingDirectory: session.installDir, skillsBaseUrl: boot.skillsBaseUrl, @@ -411,85 +238,33 @@ export const piBackend: AgentHarness = { agentDir: getAgentDir(), securityFactory: security.factory as (pi: unknown) => void, bashTool: scrubbedBash, - sdk: { createAgentSession, DefaultResourceLoader, SessionManager }, + sdk, }), ]; - const { session: agentSession } = await createAgentSession({ + const agentSession = await createHermeticSession(sdk, { + cwd: session.installDir, + systemPrompt: + getWizardCommandments() + + '\n' + + PI_RUNTIME_NOTES + + '\n' + + piMcpContext(boot, mcp.instructions), + extensionFactories, model, - modelRegistry: registry, + registry, // Reasoning effort from the switchboard capability matrix (undefined = // pi's default). Sent as `reasoning_effort` for openai-completions. thinkingLevel: caps.thinkingLevel, - cwd: session.installDir, - sessionManager: SessionManager.inMemory(session.installDir), - resourceLoader, - // Disable the default built-in tools; `customTools` re-registers - // read/edit/write + an env-scrubbed bash, so no subprocess inherits the - // host env. Custom + extension tools stay enabled. - noTools: 'builtin', customTools, }); - // Fire the extension lifecycle — what interactive mode does via - // rebindCurrentSession. createAgentSession builds the session but does not - // emit session_start on its own, and the MCP adapter connects on that - // event; without this its tools report "MCP not initialized". - await agentSession.bindExtensions({}); - - // Map pi events onto the run spinner + the log file, mirroring the - // anthropic path's log shape (assistant turns + tool I/O) and driving the - // single run spinner with one stable status at a time (no overlap). - const unsubscribe = agentSession.subscribe((event) => { - switch (event.type) { - case 'message_end': { - // User prompts also emit message_end; only assistant turns count. - if ((event.message as { role?: string })?.role !== 'assistant') { - break; - } - assistantTurns += 1; - const assistant = extractText(event.message).trim(); - if (assistant) { - logToFile(`[pi] assistant: ${assistant.slice(0, 1000)}`); - applyOutroMarkers(assistant); - // Surface [STATUS] lines into the live spinner + status history, - // mirroring the anthropic path — pi otherwise drops them. - const statusText = lastStatusLine(assistant); - if (statusText) { - getUI().pushStatus(statusText); - spinner.message(statusText); - } - for (const line of assistant.split('\n')) signals.push(line); - } - break; - } - case 'tool_execution_start': { - toolCalls += 1; - const args = JSON.stringify(event.args ?? {}).slice(0, 200); - logToFile(`[pi] → ${event.toolName} ${args}`); - // Don't surface raw tool names in the spinner — the anthropic path - // doesn't, and it reads as noise. The Task panel (syncTodos) is the - // visible progress, matching the anthropic presentation. - break; - } - case 'tool_execution_end': { - if (event.isError) { - logToFile( - `[pi] ✗ ${event.toolName}: ${String(event.result).slice( - 0, - 300, - )}`, - ); - } - break; - } - case 'agent_end': { - logToFile(`[pi] agent_end (willRetry=${String(event.willRetry)})`); - break; - } - default: - break; - } + // Map pi events onto the run spinner + the log file; counts drive the + // no-progress guard below. + const { counts, unsubscribe } = watchSession(agentSession, { + tag: '[pi]', + spinner, + signals, }); try { @@ -540,14 +315,17 @@ export const piBackend: AgentHarness = { // pi ends a run on any tool-call-less turn, so guard against a hollow // success reaching the outro (nothing done, or stopped mid-plan). const openTasks = hasOpenTasks(wizardTaskTools.store); - const failure = completionFailure({ toolCalls, openTasks }); + const failure = completionFailure({ + toolCalls: counts.toolCalls, + openTasks, + }); if (failure === AgentErrorType.NO_PROGRESS) { spinner.stop('Agent made no changes'); logToFile( - `[pi] no progress: ${assistantTurns} assistant turn(s), 0 tool calls`, + `[pi] no progress: ${counts.assistantTurns} assistant turn(s), 0 tool calls`, ); analytics.wizardCapture('agent no progress', { - assistant_turns: assistantTurns, + assistant_turns: counts.assistantTurns, }); captureAborted(); return { error: failure }; @@ -584,32 +362,22 @@ export const piBackend: AgentHarness = { logToFile(`[pi] .posthog-events.json cleanup skipped: ${String(err)}`); } - const stats = agentSession.getSessionStats(); - analytics.wizardCapture('agent completed', { - ...runDurations(), - model: modelId, - num_turns: assistantTurns, - // API-reported tokens only; no total_cost_usd — the API returns no - // cost, and $ai_generation already prices the run authoritatively. - input_tokens: stats.tokens.input, - output_tokens: stats.tokens.output, - cache_creation_input_tokens: stats.tokens.cacheWrite, - cache_read_input_tokens: stats.tokens.cacheRead, + captureAgentUsage({ + tag: '[pi]', + modelId, + counts, + durations: runDurations(), + stats: agentSession.getSessionStats(), }); spinner.stop(config.successMessage ?? 'PostHog integration complete'); return {}; } catch (err) { - const message = err instanceof Error ? err.message : String(err); + const { error, message } = classifyRunError(err); logToFile(`[pi] run error: ${message}`); spinner.stop(config.errorMessage ?? `${config.integrationLabel} failed`); getUI().log.error(`pi backend error: ${message}`); captureAborted(); - - const lower = message.toLowerCase(); - if (lower.includes('rate limit') || lower.includes('429')) { - return { error: AgentErrorType.RATE_LIMIT, message }; - } - return { error: AgentErrorType.API_ERROR, message }; + return { error, message }; } }, diff --git a/src/lib/agent/runner/harness/pi/shared.ts b/src/lib/agent/runner/harness/pi/shared.ts new file mode 100644 index 000000000..03bbe90fd --- /dev/null +++ b/src/lib/agent/runner/harness/pi/shared.ts @@ -0,0 +1,408 @@ +/** + * The pi session machinery shared by the harness's two entry points: the + * linear `run()` (index.ts) and the orchestrator's per-task `runTask()` + * (task.ts). One copy of the gateway/registry wiring, the hermetic resource + * loader, the coding-tool definitions, the session event handler, and the + * error/usage bookkeeping — so the two runs configure the same machinery + * instead of drifting apart (the 429 sniff and the analytics shape especially). + * + * No typebox in this module graph: index.ts imports it statically (the + * CommonJS unit-test seam loads it), while the SDK itself always arrives as + * the caller's lazy import, passed in as `sdk`. + */ + +import { getUI, type SpinnerHandle } from '@ui'; +import { logToFile } from '@utils/debug'; +import { analytics } from '@utils/analytics'; +import { WIZARD_USER_AGENT } from '@lib/constants'; +import { AgentErrorType } from '@lib/agent/agent-interface'; +import { AgentSignals } from '@lib/agent/signals'; +import type { AgentOutputSignals } from '@lib/agent/output-signals'; +import type { BootstrapResult } from '@lib/agent/runner/shared/types'; +import type { ThinkingLevel } from '../../switchboard/models'; +import { buildGatewayProvider, GATEWAY_PROVIDER } from './gateway'; + +/** The lazily imported pi SDK module, passed in by the caller. */ +export type PiSdk = typeof import('@earendil-works/pi-coding-agent'); +export type PiAgentSession = Awaited< + ReturnType +>['session']; +type PiModelRegistry = ReturnType; +type PiModel = NonNullable>; + +/** + * The ONLY environment variables pi's tool subprocesses (bash → npm/pip/…) are + * allowed to see. Everything else — every secret (POSTHOG_PERSONAL_API_KEY, + * ANTHROPIC_*, AWS_*), every ambient credential, the parent process's whole env + * — is dropped before a child is spawned. pi's own gateway auth is programmatic + * (the access token never lives in env), so a minimal env costs the agent + * nothing while closing the leak that exposed the key before. Kept to what a + * package manager genuinely needs to run. + */ +const ALLOWED_SUBPROCESS_ENV_KEYS = [ + 'PATH', + 'HOME', + 'SHELL', + 'USER', + 'LOGNAME', + 'TMPDIR', + 'TMP', + 'TEMP', + 'TERM', + 'LANG', + 'LC_ALL', + 'LC_CTYPE', + 'NODE_EXTRA_CA_CERTS', + 'SSL_CERT_FILE', + 'SSL_CERT_DIR', + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'NO_PROXY', + 'http_proxy', + 'https_proxy', + 'no_proxy', +]; + +/** A fresh subprocess env holding only the allowlisted keys present in process.env. */ +export function buildScrubbedEnv(): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const key of ALLOWED_SUBPROCESS_ENV_KEYS) { + const value = process.env[key]; + if (value !== undefined) env[key] = value; + } + return env; +} + +/** + * Tag a tool with an execution mode (mutates + returns it). Read-only tools are + * `parallel` so a single turn that batches independent reads/searches runs them + * at once; mutating/install tools are `sequential` so a batch never races writes + * or concurrent installs. pi-agent-core runs a batch in parallel only when no + * tool in it is `sequential`. + */ +export function withMode(tool: T, mode: 'sequential' | 'parallel'): T { + (tool as { executionMode?: 'sequential' | 'parallel' }).executionMode = mode; + return tool; +} + +/** Pull plain text out of a pi AgentMessage (content is text/image blocks). */ +export function extractText(message: unknown): string { + const content = (message as { content?: unknown })?.content; + if (typeof content === 'string') return content; + if (Array.isArray(content)) { + return content + .filter((c): c is { type: string; text: string } => { + const block = c as { type?: string; text?: unknown }; + return block?.type === 'text' && typeof block.text === 'string'; + }) + .map((c) => c.text) + .join(''); + } + return ''; +} + +/** + * Surface `[DASHBOARD_URL]` / `[NOTEBOOK_URL]` markers the agent prints (after + * the MCP creates them) into the outro link, mirroring the anthropic path's + * signal parsing (#9). The marker carries the URL the MCP returned. + */ +export function applyOutroMarkers(textBlock: string): void { + const markers: Array<[string, (url: string) => void]> = [ + [AgentSignals.DASHBOARD_URL, (url) => getUI().setDashboardUrl(url)], + [AgentSignals.NOTEBOOK_URL, (url) => getUI().setNotebookUrl(url)], + ]; + for (const [marker, apply] of markers) { + const idx = textBlock.indexOf(marker); + if (idx === -1) continue; + const url = textBlock + .slice(idx + marker.length) + .trim() + .split(/\s/)[0]; + if (url) apply(url); + } +} + +/** + * The text of the last `[STATUS] …` line in a block, if any. Last wins so the + * spinner shows the most recent action when a turn prints several. + */ +export function lastStatusLine(textBlock: string): string | undefined { + let status: string | undefined; + for (const line of textBlock.split('\n')) { + const idx = line.indexOf(AgentSignals.STATUS); + if (idx !== -1) { + status = line.slice(idx + AgentSignals.STATUS.length).trim(); + } + } + return status || undefined; +} + +/** + * Register the PostHog gateway on a fresh in-memory registry and resolve the + * model. Undefined when the model can't be resolved — the caller returns its + * API_ERROR. `applyEffortFlag`/`effort` follow `buildGatewayProvider`: + * the linear run honours the wizard-pi-effort flag, a per-task run passes + * `applyEffortFlag: false` and its own frontmatter effort. + */ +export function resolveGatewayModel( + sdk: PiSdk, + boot: BootstrapResult, + modelId: string, + opts: { applyEffortFlag?: boolean; effort?: ThinkingLevel } = {}, +): + | { + registry: PiModelRegistry; + model: PiModel; + caps: ReturnType['caps']; + gatewayUrl: string; + } + | undefined { + const { provider, caps, gatewayUrl } = buildGatewayProvider({ + gatewayUrl: boot.credentials.host.gatewayUrl, + accessToken: boot.credentials.accessToken, + wizardMetadata: boot.wizardMetadata, + wizardFlags: boot.wizardFlags, + modelId, + applyEffortFlag: opts.applyEffortFlag, + effort: opts.effort, + }); + const registry = sdk.ModelRegistry.inMemory(sdk.AuthStorage.create()); + registry.registerProvider(GATEWAY_PROVIDER, provider as never); + const model = registry.find(GATEWAY_PROVIDER, modelId); + if (!model) return undefined; + return { registry, model, caps, gatewayUrl }; +} + +/** + * Wire the real PostHog MCP, best-effort: if the adapter can't load or + * connect, the run continues (minus the posthog_* tools) rather than failing. + * The caller pushes `extensionFactory` after the security factory and calls + * `cleanup` when the session ends. + */ +export async function connectPostHogMcp( + sdk: PiSdk, + boot: BootstrapResult, + tag: string, +): Promise<{ + extensionFactory?: (pi: unknown) => void; + cleanup?: () => void; + instructions?: string; +}> { + try { + const { setupPostHogMcp } = await import('./mcp'); + return await setupPostHogMcp({ + agentDir: sdk.getAgentDir(), + mcpUrl: boot.credentials.host.mcpUrl, + accessToken: boot.credentials.accessToken, + userAgent: WIZARD_USER_AGENT, + }); + } catch (err) { + logToFile(`${tag} PostHog MCP setup skipped: ${String(err)}`); + return {}; + } +} + +/** + * The stock coding tools rooted at `dir`, each tagged with its execution mode + * and bash wired to the scrubbed subprocess env. Factories, not instances, so + * a caller registers exactly the tools its allow list grants. + */ +export function piCodingToolFactories(sdk: PiSdk, dir: string) { + return { + read: () => withMode(sdk.createReadToolDefinition(dir), 'parallel'), + edit: () => withMode(sdk.createEditToolDefinition(dir), 'sequential'), + write: () => withMode(sdk.createWriteToolDefinition(dir), 'sequential'), + bash: () => + withMode( + sdk.createBashToolDefinition(dir, { + spawnHook: (ctx) => ({ ...ctx, env: buildScrubbedEnv() }), + }), + 'sequential', + ), + ls: () => withMode(sdk.createLsToolDefinition(dir), 'parallel'), + find: () => withMode(sdk.createFindToolDefinition(dir), 'parallel'), + grep: () => withMode(sdk.createGrepToolDefinition(dir), 'parallel'), + } as const; +} + +/** + * A hermetic pi session: the given system prompt and extensions only — no + * disk-discovered extensions, skills, context files, prompt templates, or + * themes from the target project — with pi's default built-in tools disabled + * so `customTools` is the entire tool surface. Fires the extension lifecycle + * (`bindExtensions`) before returning, which the MCP adapter connects on. + */ +export async function createHermeticSession( + sdk: PiSdk, + opts: { + cwd: string; + systemPrompt: string; + extensionFactories: Array<(pi: unknown) => void>; + model: PiModel; + registry: PiModelRegistry; + thinkingLevel?: ThinkingLevel; + customTools: Parameters[0]['customTools']; + }, +): Promise { + const resourceLoader = new sdk.DefaultResourceLoader({ + cwd: opts.cwd, + agentDir: sdk.getAgentDir(), + systemPrompt: opts.systemPrompt, + noExtensions: true, + noSkills: true, + noContextFiles: true, + noPromptTemplates: true, + noThemes: true, + extensionFactories: opts.extensionFactories, + }); + await resourceLoader.reload(); + + const { session } = await sdk.createAgentSession({ + model: opts.model, + modelRegistry: opts.registry, + thinkingLevel: opts.thinkingLevel, + cwd: opts.cwd, + sessionManager: sdk.SessionManager.inMemory(opts.cwd), + resourceLoader, + noTools: 'builtin', + customTools: opts.customTools, + }); + await session.bindExtensions({}); + return session; +} + +/** Counters the session watcher keeps live; read them after the run. */ +export interface SessionCounts { + assistantTurns: number; + toolCalls: number; +} + +/** + * Map pi session events onto the run spinner + the log file, mirroring the + * anthropic path's log shape (assistant turns + tool I/O) and driving the + * single run spinner with one stable status at a time. `[STATUS]` lines + * surface into the live spinner and status history; outro markers apply as + * they stream; every assistant line feeds `signals`. + */ +export function watchSession( + agentSession: PiAgentSession, + opts: { tag: string; spinner: SpinnerHandle; signals: AgentOutputSignals }, +): { counts: SessionCounts; unsubscribe: () => void } { + const { tag, spinner, signals } = opts; + const counts: SessionCounts = { assistantTurns: 0, toolCalls: 0 }; + const unsubscribe = agentSession.subscribe((event) => { + switch (event.type) { + case 'message_end': { + // User prompts also emit message_end; only assistant turns count. + if ((event.message as { role?: string })?.role !== 'assistant') { + break; + } + counts.assistantTurns += 1; + const assistant = extractText(event.message).trim(); + if (assistant) { + logToFile(`${tag} assistant: ${assistant.slice(0, 1000)}`); + applyOutroMarkers(assistant); + const statusText = lastStatusLine(assistant); + if (statusText) { + getUI().pushStatus(statusText); + spinner.message(statusText); + } + for (const line of assistant.split('\n')) signals.push(line); + } + break; + } + case 'tool_execution_start': { + counts.toolCalls += 1; + const args = JSON.stringify(event.args ?? {}).slice(0, 200); + logToFile(`${tag} → ${event.toolName} ${args}`); + // Don't surface raw tool names in the spinner — the anthropic path + // doesn't, and it reads as noise. + break; + } + case 'tool_execution_end': { + if (event.isError) { + logToFile( + `${tag} ✗ ${event.toolName}: ${String(event.result).slice(0, 300)}`, + ); + } + break; + } + case 'agent_end': { + logToFile(`${tag} agent_end (willRetry=${String(event.willRetry)})`); + break; + } + default: + break; + } + }); + return { counts, unsubscribe }; +} + +/** A run clock: call once at start, call the result for the durations shape. */ +export function startRunClock(): () => { + duration_ms: number; + duration_seconds: number; +} { + const startTime = Date.now(); + return () => { + const durationMs = Date.now() - startTime; + return { + duration_ms: durationMs, + duration_seconds: Math.round(durationMs / 1000), + }; + }; +} + +/** Classify a thrown run error the way both entry points report it. */ +export function classifyRunError(err: unknown): { + error: AgentErrorType; + message: string; +} { + const message = err instanceof Error ? err.message : String(err); + const lower = message.toLowerCase(); + const error = + lower.includes('rate limit') || lower.includes('429') + ? AgentErrorType.RATE_LIMIT + : AgentErrorType.API_ERROR; + return { error, message }; +} + +/** + * The `agent completed` capture both entry points send, plus one parseable + * usage line in the log so a run's per-unit time and cost are observable + * without analytics access. `task=` appears only when the run carries a + * task_type (orchestrator units); the linear run has none. + */ +export function captureAgentUsage(opts: { + tag: string; + modelId: string; + counts: SessionCounts; + durations: { duration_ms: number; duration_seconds: number }; + stats: ReturnType; + analyticsProperties?: Record; +}): void { + const { tag, modelId, counts, durations, stats, analyticsProperties } = opts; + analytics.wizardCapture('agent completed', { + ...durations, + model: modelId, + num_turns: counts.assistantTurns, + // API-reported tokens only; no total_cost_usd — the API returns no + // cost, and $ai_generation already prices the run authoritatively. + input_tokens: stats.tokens.input, + output_tokens: stats.tokens.output, + cache_creation_input_tokens: stats.tokens.cacheWrite, + cache_read_input_tokens: stats.tokens.cacheRead, + ...analyticsProperties, + }); + const taskType = + typeof analyticsProperties?.task_type === 'string' + ? analyticsProperties.task_type + : undefined; + logToFile( + `${tag} usage${taskType ? ` task=${taskType}` : ''} model=${modelId} dur=${ + durations.duration_seconds + }s turns=${counts.assistantTurns} in=${stats.tokens.input} out=${ + stats.tokens.output + } cacheR=${stats.tokens.cacheRead} cacheW=${stats.tokens.cacheWrite}`, + ); +} diff --git a/src/lib/agent/runner/harness/pi/subagent.ts b/src/lib/agent/runner/harness/pi/subagent.ts index 1f5e7f7d6..477c884ec 100644 --- a/src/lib/agent/runner/harness/pi/subagent.ts +++ b/src/lib/agent/runner/harness/pi/subagent.ts @@ -17,6 +17,7 @@ import { Type } from 'typebox'; import { defineTool } from '@earendil-works/pi-coding-agent'; import type { ToolDefinition } from '@earendil-works/pi-coding-agent'; import { logToFile } from '@utils/debug'; +import { extractText } from './shared'; /** * Read-only built-ins a subagent may use. bash is supplied separately as the @@ -39,21 +40,6 @@ function text(s: string): { return { content: [{ type: 'text', text: s }], details: {} }; } -function extractText(message: unknown): string { - const content = (message as { content?: unknown })?.content; - if (typeof content === 'string') return content; - if (Array.isArray(content)) { - return content - .filter((c): c is { type: string; text: string } => { - const b = c as { type?: string; text?: unknown }; - return b?.type === 'text' && typeof b.text === 'string'; - }) - .map((c) => c.text) - .join(''); - } - return ''; -} - export interface SubagentContext { /** Resolved gateway model (same as the parent). */ // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/src/lib/agent/runner/harness/pi/task.ts b/src/lib/agent/runner/harness/pi/task.ts index de9f72685..e17a25719 100644 --- a/src/lib/agent/runner/harness/pi/task.ts +++ b/src/lib/agent/runner/harness/pi/task.ts @@ -1,10 +1,11 @@ /** * Orchestrator-mode execution on pi: one fresh pi session per unit of work — - * the seed plan, or one drained task. The linear pipeline's concerns (skill - * menu, todo panel, event-plan cleanup) stay in `index.ts`; this module builds - * the leaner per-task session: gateway model, security fence, the task's - * allowed coding tools, the wizard env tools, and the in-process orchestrator - * queue tools. + * the seed plan, or one drained task. The session machinery (gateway model, + * hermetic loader, event watching, error/usage bookkeeping) is `shared.ts`, + * common with the linear run; the linear pipeline's own concerns (skill menu, + * todo panel, event-plan cleanup) stay in `index.ts`. This module configures + * the leaner per-task session: the task's allowed coding tools, the wizard env + * tools, and the in-process orchestrator queue tools. * * The task's `allowedTools` / `disallowedTools` arrive in the wizard's tool * vocabulary (`Read`, `Edit`, `Glob`, …, plus MCP-qualified orchestrator names @@ -16,24 +17,25 @@ * Loaded lazily from `index.ts` (typebox/ESM constraint, same as tools.ts). */ -import { getUI } from '@ui'; import { logToFile } from '@utils/debug'; import { analytics } from '@utils/analytics'; -import { WIZARD_REMARK_EVENT_NAME, WIZARD_USER_AGENT } from '@lib/constants'; +import { WIZARD_REMARK_EVENT_NAME } from '@lib/constants'; import { AgentErrorType } from '@lib/agent/agent-interface'; import { REMARK_INSTRUCTION } from '@lib/agent/signals'; import { AgentOutputSignals } from '@lib/agent/output-signals'; import { TaskStatus } from '../../sequence/orchestrator/queue'; import type { OrchestratorToolsContext } from '../../sequence/orchestrator/queue-tools'; import type { AgentResult, TaskRunInputs } from '../types'; -import { buildGatewayProvider, GATEWAY_PROVIDER } from './gateway'; import { - applyOutroMarkers, - buildScrubbedEnv, - extractText, - lastStatusLine, - withMode, -} from './index'; + captureAgentUsage, + classifyRunError, + connectPostHogMcp, + createHermeticSession, + piCodingToolFactories, + resolveGatewayModel, + startRunClock, + watchSession, +} from './shared'; /** wizard tool vocabulary → the pi tool definitions it unlocks. */ const CODING_TOOL_MAP: Record = { @@ -167,16 +169,8 @@ export async function runPiTask(inputs: TaskRunInputs): Promise { if (spinnerMessage) spinner.start(spinnerMessage); - const startTime = Date.now(); const signals = new AgentOutputSignals(); - let assistantTurns = 0; - const runDurations = () => { - const durationMs = Date.now() - startTime; - return { - duration_ms: durationMs, - duration_seconds: Math.round(durationMs / 1000), - }; - }; + const runDurations = startRunClock(); const captureAborted = () => analytics.wizardCapture('agent aborted', { ...runDurations(), @@ -186,42 +180,20 @@ export async function runPiTask(inputs: TaskRunInputs): Promise { try { const sdk = await import('@earendil-works/pi-coding-agent'); - const { - createAgentSession, - DefaultResourceLoader, - SessionManager, - AuthStorage, - ModelRegistry, - getAgentDir, - createLsToolDefinition, - createFindToolDefinition, - createGrepToolDefinition, - createBashToolDefinition, - createReadToolDefinition, - createEditToolDefinition, - createWriteToolDefinition, - } = sdk; - const { provider, caps, gatewayUrl } = buildGatewayProvider({ - gatewayUrl: boot.credentials.host.gatewayUrl, - accessToken: boot.credentials.accessToken, - wizardMetadata: boot.wizardMetadata, - wizardFlags: boot.wizardFlags, - modelId, - // Per-task agents own their effort via the prompt frontmatter (falling back - // to the model table), not the run-wide wizard-pi-effort flag. + // Per-task agents own their effort via the prompt frontmatter (falling + // back to the model table), not the run-wide wizard-pi-effort flag. + const gateway = resolveGatewayModel(sdk, boot, modelId, { applyEffortFlag: false, effort, }); - const registry = ModelRegistry.inMemory(AuthStorage.create()); - registry.registerProvider(GATEWAY_PROVIDER, provider as never); - const model = registry.find(GATEWAY_PROVIDER, modelId); - if (!model) { + if (!gateway) { return { error: AgentErrorType.API_ERROR, message: 'pi: gateway model could not be resolved', }; } + const { registry, model, caps, gatewayUrl } = gateway; // The same fail-closed fence as the linear run, with the task's disallow // list layered in (both the wizard-vocabulary and pi-short names). @@ -241,63 +213,19 @@ export async function runPiTask(inputs: TaskRunInputs): Promise { const extensionFactories = [security.factory] as Array< (pi: unknown) => void >; - let mcpCleanup: (() => void) | undefined; - let posthogMcp = false; - try { - const { setupPostHogMcp } = await import('./mcp'); - const mcp = await setupPostHogMcp({ - agentDir: getAgentDir(), - mcpUrl: boot.credentials.host.mcpUrl, - accessToken: boot.credentials.accessToken, - userAgent: WIZARD_USER_AGENT, - }); - extensionFactories.push(mcp.extensionFactory); - mcpCleanup = mcp.cleanup; - posthogMcp = true; - } catch (err) { - logToFile(`[pi-task] PostHog MCP setup skipped: ${String(err)}`); - } + const mcp = await connectPostHogMcp(sdk, boot, '[pi-task]'); + if (mcp.extensionFactory) extensionFactories.push(mcp.extensionFactory); + const mcpCleanup = mcp.cleanup; + const posthogMcp = mcp.extensionFactory !== undefined; const codingTools = allowedPiCodingTools(allowedTools); const orchestratorTools = allowedOrchestratorTools(disallowedTools); - const { getWizardCommandments } = await import('@lib/agent/commandments'); - const resourceLoader = new DefaultResourceLoader({ - cwd: session.installDir, - agentDir: getAgentDir(), - systemPrompt: - getWizardCommandments() + - '\n' + - taskRuntimeNotes({ bash: codingTools.has('bash'), posthogMcp }), - noExtensions: true, - noSkills: true, - noContextFiles: true, - noPromptTemplates: true, - noThemes: true, - extensionFactories, - }); - await resourceLoader.reload(); - // The task's coding tools, gated by its allow list. Reads and searches run // in parallel; mutating tools stay sequential. Bash subprocesses get the // scrubbed env, same as the linear run. const dir = session.installDir; - const codingToolFactories = { - read: () => withMode(createReadToolDefinition(dir), 'parallel'), - edit: () => withMode(createEditToolDefinition(dir), 'sequential'), - write: () => withMode(createWriteToolDefinition(dir), 'sequential'), - bash: () => - withMode( - createBashToolDefinition(dir, { - spawnHook: (ctx) => ({ ...ctx, env: buildScrubbedEnv() }), - }), - 'sequential', - ), - ls: () => withMode(createLsToolDefinition(dir), 'parallel'), - find: () => withMode(createFindToolDefinition(dir), 'parallel'), - grep: () => withMode(createGrepToolDefinition(dir), 'parallel'), - } as const; - const codingToolDefs = Object.entries(codingToolFactories) + const codingToolDefs = Object.entries(piCodingToolFactories(sdk, dir)) .filter(([name]) => codingTools.has(name)) .map(([, make]) => make()); @@ -318,58 +246,24 @@ export async function runPiTask(inputs: TaskRunInputs): Promise { orchestratorTools.has(t.name), ); - const { session: agentSession } = await createAgentSession({ + const { getWizardCommandments } = await import('@lib/agent/commandments'); + const agentSession = await createHermeticSession(sdk, { + cwd: dir, + systemPrompt: + getWizardCommandments() + + '\n' + + taskRuntimeNotes({ bash: codingTools.has('bash'), posthogMcp }), + extensionFactories, model, - modelRegistry: registry, + registry, thinkingLevel: caps.thinkingLevel, - cwd: dir, - sessionManager: SessionManager.inMemory(dir), - resourceLoader, - noTools: 'builtin', customTools: [...codingToolDefs, ...wizardTools, ...queueTools], }); - await agentSession.bindExtensions({}); - const unsubscribe = agentSession.subscribe((event) => { - switch (event.type) { - case 'message_end': { - // User prompts also emit message_end; only assistant turns count. - if ((event.message as { role?: string })?.role !== 'assistant') { - break; - } - assistantTurns += 1; - const assistant = extractText(event.message).trim(); - if (assistant) { - logToFile(`[pi-task] assistant: ${assistant.slice(0, 1000)}`); - applyOutroMarkers(assistant); - const statusText = lastStatusLine(assistant); - if (statusText) { - getUI().pushStatus(statusText); - spinner.message(statusText); - } - for (const line of assistant.split('\n')) signals.push(line); - } - break; - } - case 'tool_execution_start': { - const args = JSON.stringify(event.args ?? {}).slice(0, 200); - logToFile(`[pi-task] → ${event.toolName} ${args}`); - break; - } - case 'tool_execution_end': { - if (event.isError) { - logToFile( - `[pi-task] ✗ ${event.toolName}: ${String(event.result).slice( - 0, - 300, - )}`, - ); - } - break; - } - default: - break; - } + const { counts, unsubscribe } = watchSession(agentSession, { + tag: '[pi-task]', + spinner, + signals, }); try { @@ -419,41 +313,23 @@ export async function runPiTask(inputs: TaskRunInputs): Promise { analytics.capture(WIZARD_REMARK_EVENT_NAME, { remark }); } - const stats = agentSession.getSessionStats(); - const durations = runDurations(); - analytics.wizardCapture('agent completed', { - ...durations, - model: modelId, - num_turns: assistantTurns, - input_tokens: stats.tokens.input, - output_tokens: stats.tokens.output, - cache_creation_input_tokens: stats.tokens.cacheWrite, - cache_read_input_tokens: stats.tokens.cacheRead, - ...analyticsProperties, + captureAgentUsage({ + tag: '[pi-task]', + modelId, + counts, + durations: runDurations(), + stats: agentSession.getSessionStats(), + analyticsProperties, }); - // Per-task usage on one parseable line so a run's per-task time and cost are - // observable from the log, not only from analytics. - const taskType = - typeof (analyticsProperties as { task_type?: unknown })?.task_type === - 'string' - ? (analyticsProperties as { task_type: string }).task_type - : modelId; - logToFile( - `[pi-task] usage task=${taskType} model=${modelId} dur=${durations.duration_seconds}s turns=${assistantTurns} in=${stats.tokens.input} out=${stats.tokens.output} cacheR=${stats.tokens.cacheRead} cacheW=${stats.tokens.cacheWrite}`, - ); if (successMessage) spinner.stop(successMessage); return {}; } catch (err) { - const message = err instanceof Error ? err.message : String(err); + const { error, message } = classifyRunError(err); logToFile(`[pi-task] run error: ${message}`); if (errorMessage || spinnerMessage) { spinner.stop(errorMessage ?? 'Task failed'); } captureAborted(); - const lower = message.toLowerCase(); - if (lower.includes('rate limit') || lower.includes('429')) { - return { error: AgentErrorType.RATE_LIMIT, message }; - } - return { error: AgentErrorType.API_ERROR, message }; + return { error, message }; } } diff --git a/src/lib/agent/runner/harness/types.ts b/src/lib/agent/runner/harness/types.ts index 2687a57e6..5efaa7733 100644 --- a/src/lib/agent/runner/harness/types.ts +++ b/src/lib/agent/runner/harness/types.ts @@ -23,6 +23,7 @@ import type { SpinnerHandle } from '@ui'; import type { WizardAskBridge } from '@lib/wizard-ask-bridge'; import type { AgentErrorType } from '@lib/agent/agent-interface'; import type { OrchestratorToolsContext } from '@lib/agent/runner/sequence/orchestrator/queue-tools'; +import type { ThinkingLevel } from '@lib/agent/runner/switchboard/models'; import type { ProgramRun, BootstrapResult, @@ -79,7 +80,7 @@ export interface TaskRunInputs { model: string; /** Reasoning effort from the agent prompt's per-profile frontmatter; overrides * the model's table default when set. */ - effort?: string; + effort?: ThinkingLevel; /** Per-task tool overrides from the agent prompt's frontmatter. */ allowedTools?: readonly string[]; disallowedTools?: readonly string[]; diff --git a/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts b/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts index 0f2106f1a..9367bed4d 100644 --- a/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts +++ b/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts @@ -1,68 +1,148 @@ import { describe, it, expect } from 'vitest'; import { resolveSkillVariantId } from '../orchestrator-runner'; import { Integration } from '@lib/constants'; +import type { SkillEntry } from '@lib/wizard-tools'; -// A representative slice of the real install-step menu ids, including the -// families whose variant id differs from the framework enum. -const MENU = [ - 'posthog-integration-install-django', - 'posthog-integration-install-laravel', - 'posthog-integration-install-nextjs-app-router', - 'posthog-integration-install-nextjs-pages-router', - 'posthog-integration-install-nuxt-3-6', - 'posthog-integration-install-vue-3', - 'posthog-integration-install-astro-hybrid', - 'posthog-integration-install-ruby-on-rails', - 'posthog-integration-install-react-react-router-6', - 'posthog-integration-install-react-react-router-7-framework', - 'posthog-integration-install-react-tanstack-router-code-based', - 'posthog-integration-install-swift', - 'posthog-integration-install-javascript_web', +// The integration group exactly as the context-mill build emits it into +// skill-menu.json (id + group + framework + default), so this suite pins the +// real cross-repo contract instead of a fixture that mirrors the resolver. +const INTEGRATION_ENTRIES = [ + { id: 'integration-nextjs-app-router', framework: 'nextjs', default: true }, + { id: 'integration-nextjs-pages-router', framework: 'nextjs' }, + { + id: 'integration-react-react-router-6', + framework: 'react-router', + default: true, + }, + { + id: 'integration-react-react-router-7-framework', + framework: 'react-router', + }, + { id: 'integration-react-react-router-7-data', framework: 'react-router' }, + { + id: 'integration-react-react-router-7-declarative', + framework: 'react-router', + }, + { id: 'integration-react-vite' }, + { id: 'integration-nuxt-3-6', framework: 'nuxt', default: true }, + { id: 'integration-nuxt-4', framework: 'nuxt' }, + { id: 'integration-vue-3', framework: 'vue' }, + { id: 'integration-django', framework: 'django' }, + { id: 'integration-flask', framework: 'flask' }, + { id: 'integration-fastapi', framework: 'fastapi' }, + { + id: 'integration-react-tanstack-router-file-based', + framework: 'tanstack-router', + }, + { + id: 'integration-react-tanstack-router-code-based', + framework: 'tanstack-router', + default: true, + }, + { id: 'integration-tanstack-start', framework: 'tanstack-start' }, + { id: 'integration-laravel', framework: 'laravel' }, + { id: 'integration-php' }, + { id: 'integration-ruby-on-rails', framework: 'rails' }, + { id: 'integration-android', framework: 'android' }, + { id: 'integration-sveltekit', framework: 'sveltekit' }, + { id: 'integration-python', framework: 'python' }, + { id: 'integration-javascript_node', framework: 'javascript_node' }, + { id: 'integration-javascript_web', framework: 'javascript_web' }, + { id: 'integration-ruby', framework: 'ruby' }, + { id: 'integration-elixir' }, + { id: 'integration-go' }, + { id: 'integration-swift', framework: 'swift' }, + { id: 'integration-flutter' }, + { id: 'integration-react-native', framework: 'react-native', default: true }, + { id: 'integration-expo', framework: 'react-native' }, + { id: 'integration-astro-static', framework: 'astro' }, + { id: 'integration-astro-view-transitions', framework: 'astro' }, + { id: 'integration-astro-ssr', framework: 'astro' }, + { id: 'integration-astro-hybrid', framework: 'astro', default: true }, + { id: 'integration-angular', framework: 'angular' }, +].map( + (e): SkillEntry => ({ + ...e, + group: 'integration', + name: e.id, + downloadUrl: `https://example.test/${e.id}.zip`, + }), +); + +// A single-variant skill collapses to the bare group id in the menu. +const MENU: SkillEntry[] = [ + ...INTEGRATION_ENTRIES, + { + id: 'posthog-integration-build', + group: 'posthog-integration-build', + name: 'build', + downloadUrl: 'https://example.test/posthog-integration-build.zip', + }, ]; -const SKILL = 'posthog-integration-install'; +describe('resolveSkillVariantId — menu-declared framework resolution', () => { + it('resolves a bare single-variant skill id to itself', () => { + expect( + resolveSkillVariantId(MENU, 'posthog-integration-build', 'django'), + ).toBe('posthog-integration-build'); + }); + + it('resolves a full menu id to itself, regardless of framework', () => { + expect( + resolveSkillVariantId(MENU, 'integration-nextjs-pages-router', 'nextjs'), + ).toBe('integration-nextjs-pages-router'); + }); -describe('resolveSkillVariantId — framework/variant parity', () => { - it('resolves the enums whose variant id differs from the enum value', () => { - expect(resolveSkillVariantId(MENU, SKILL, 'rails')).toBe( - 'posthog-integration-install-ruby-on-rails', + it('resolves the frameworks whose variant id differs from the detection id', () => { + expect(resolveSkillVariantId(MENU, 'integration', 'rails')).toBe( + 'integration-ruby-on-rails', ); - expect(resolveSkillVariantId(MENU, SKILL, 'react-router')).toBe( - 'posthog-integration-install-react-react-router-6', + expect(resolveSkillVariantId(MENU, 'integration', 'react-router')).toBe( + 'integration-react-react-router-6', ); - expect(resolveSkillVariantId(MENU, SKILL, 'tanstack-router')).toBe( - 'posthog-integration-install-react-tanstack-router-code-based', + expect(resolveSkillVariantId(MENU, 'integration', 'tanstack-router')).toBe( + 'integration-react-tanstack-router-code-based', ); }); - it('still resolves the frameworks that match by id or prefix', () => { - expect(resolveSkillVariantId(MENU, SKILL, 'django')).toBe( - 'posthog-integration-install-django', + it('picks the marked default when a family has several variants', () => { + expect(resolveSkillVariantId(MENU, 'integration', 'nextjs')).toBe( + 'integration-nextjs-app-router', + ); + expect(resolveSkillVariantId(MENU, 'integration', 'astro')).toBe( + 'integration-astro-hybrid', ); - expect(resolveSkillVariantId(MENU, SKILL, 'nextjs')).toBe( - 'posthog-integration-install-nextjs-app-router', + expect(resolveSkillVariantId(MENU, 'integration', 'react-native')).toBe( + 'integration-react-native', ); - expect(resolveSkillVariantId(MENU, SKILL, 'vue')).toBe( - 'posthog-integration-install-vue-3', + }); + + it('resolves a single-entry family without needing a default marker', () => { + expect(resolveSkillVariantId(MENU, 'integration', 'vue')).toBe( + 'integration-vue-3', + ); + expect(resolveSkillVariantId(MENU, 'integration', 'django')).toBe( + 'integration-django', + ); + }); + + it('returns undefined without a framework or without a matching entry', () => { + expect( + resolveSkillVariantId(MENU, 'integration', undefined), + ).toBeUndefined(); + expect(resolveSkillVariantId(MENU, 'integration', 'cobol')).toBeUndefined(); + // A variant with no framework field (react-vite) is only reachable by id. + expect(resolveSkillVariantId(MENU, 'integration-react-vite', 'vue')).toBe( + 'integration-react-vite', ); }); - it('every framework in a full menu resolves — no silent zero-diff', () => { - // A menu with one install variant per Integration enum (aliased where the - // id differs), so the whole enum must resolve. - const alias: Record = { - 'react-router': 'react-react-router-7-framework', - 'tanstack-router': 'react-tanstack-router-code-based', - rails: 'ruby-on-rails', - nextjs: 'nextjs-app-router', - nuxt: 'nuxt-3-6', - vue: 'vue-3', - astro: 'astro-hybrid', - }; - const enums = Object.values(Integration); - const menu = enums.map((e) => `${SKILL}-${alias[e] ?? e}`); - for (const e of enums) { - expect(resolveSkillVariantId(menu, SKILL, e)).toBeDefined(); + it('every framework in the Integration enum resolves — no silent zero-diff', () => { + for (const framework of Object.values(Integration)) { + expect( + resolveSkillVariantId(MENU, 'integration', framework), + `framework "${framework}" resolved nothing`, + ).toBeDefined(); } }); }); diff --git a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts index 81a58b3e2..a7e7ed3e7 100644 --- a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -14,7 +14,11 @@ import { randomUUID } from 'crypto'; import { existsSync, rmSync } from 'fs'; import * as path from 'path'; import { OutroKind, type WizardSession } from '@lib/wizard-session'; -import { installSkillById, fetchSkillMenu } from '@lib/wizard-tools'; +import { + installSkillById, + fetchSkillMenu, + type SkillEntry, +} from '@lib/wizard-tools'; import { getUI } from '@ui'; import { analytics } from '@utils/analytics'; import { ciExcludedTaskTypes } from '@utils/ci-flag-overrides'; @@ -79,47 +83,35 @@ function requireTaskHarness(pick: HarnessPick): AgentHarness & { }; } -/** Every skill id the menu knows, across categories. */ -async function fetchSkillMenuIds(skillsBaseUrl: string): Promise { +/** Every skill entry the menu knows, across categories. */ +async function fetchSkillMenuEntries( + skillsBaseUrl: string, +): Promise { const menu = await fetchSkillMenu(skillsBaseUrl); if (!menu) return []; - return Object.values(menu.categories) - .flat() - .map((s) => s.id); + return Object.values(menu.categories).flat(); } /** - * Resolve a bare skill id + the session's framework to the menu id: the bare - * id itself (single-variant skills collapse to it), else exact - * `-` (the 1:1 frameworks — django, python, flask, …), else the - * first granular variant under the framework (e.g. `-nextjs-app-router`). - * Undefined when nothing matches. - */ -/** - * Framework enums whose context-mill variant id differs from the enum value. - * The orchestrator resolves variants programmatically (the linear flow's agent - * picks from the menu by hand and self-corrects), so without these it silently - * resolves nothing and the tasks run skill-less — a zero-diff run. The value is - * the variant-id token (or its prefix, for a family the `startsWith` fallback - * then narrows). + * Resolve a bare skill id + the session's framework to the menu id, from the + * `group`/`framework`/`default` fields the menu declares: a full menu id (or + * single-variant skill) resolves to itself; otherwise the entry whose group is + * the bare id and whose framework matches — the family's marked default when + * several variants serve one framework (e.g. app vs pages router). No wizard- + * side vocabulary: context-mill owns which variant serves which framework, so + * a rename there cannot silently strand the resolution here. */ -const FRAMEWORK_VARIANT_ALIASES: Record = { - rails: 'ruby-on-rails', - 'react-router': 'react-react-router', - 'tanstack-router': 'react-tanstack-router', -}; - export function resolveSkillVariantId( - menuIds: readonly string[], + entries: readonly SkillEntry[], skillId: string, framework: string | undefined, ): string | undefined { - if (menuIds.includes(skillId)) return skillId; + if (entries.some((e) => e.id === skillId)) return skillId; if (!framework) return undefined; - const variant = FRAMEWORK_VARIANT_ALIASES[framework] ?? framework; - const exact = `${skillId}-${variant}`; - if (menuIds.includes(exact)) return exact; - return menuIds.find((id) => id.startsWith(`${exact}-`)); + const family = entries.filter( + (e) => e.group === skillId && e.framework === framework, + ); + return (family.find((e) => e.default) ?? family[0])?.id; } /** @@ -128,10 +120,10 @@ export function resolveSkillVariantId( * `integration-`. */ function resolveReferenceSkillId( - menuIds: readonly string[], + entries: readonly SkillEntry[], framework: string, ): string | undefined { - return resolveSkillVariantId(menuIds, 'integration', framework); + return resolveSkillVariantId(entries, 'integration', framework); } export async function runOrchestrator( @@ -178,13 +170,10 @@ export async function runOrchestrator( const store = new QueueStore(session.installDir, runId, { onTransition: (event, task) => { + const pick = resolveHarness(switchboardCtx, task.type); const base = { type: task.type, - model: taskModelSpec( - registry, - task, - resolveHarness(switchboardCtx, task.type).harness, - ).model, + model: taskModelSpec(registry, task, pick.harness).model ?? pick.model, attempts: task.attempts, }; switch (event) { @@ -235,9 +224,9 @@ export async function runOrchestrator( // skill — only the example file is read, when the agent's prompt points at it. let examplePath: string | undefined; let commandmentsPath: string | undefined; - const menuSkillIds = await fetchSkillMenuIds(boot.skillsBaseUrl); + const menuSkillEntries = await fetchSkillMenuEntries(boot.skillsBaseUrl); const referenceSkillId = session.skillId - ? resolveReferenceSkillId(menuSkillIds, session.skillId) + ? resolveReferenceSkillId(menuSkillEntries, session.skillId) : undefined; if (referenceSkillId) { const ref = await installSkillById( @@ -266,6 +255,27 @@ export async function runOrchestrator( ); } + // Preflight the HOW: resolve every task's mini-skills against the menu now, + // so a variant gap between the registry and the menu surfaces (log + + // analytics) before any agent runs — not as a silently skill-less task + // discovered mid-drain, whose zero-diff run would look like a success. + for (const type of registry.types) { + for (const skillId of registry.get(type)?.skills ?? []) { + if (!resolveSkillVariantId(menuSkillEntries, skillId, session.skillId)) { + logToFile( + `[orchestrator] no skill variant type=${type} skill=${skillId} framework=${ + session.skillId ?? 'none' + }`, + ); + analytics.wizardCapture('orchestrator skill variant missing', { + task_type: type, + skill: skillId, + framework: session.skillId, + }); + } + } + } + // The client injects the basics (project context + the I/O contract) around // every authored agent-prompt body. const promptContext: OrchestratorPromptContext = { @@ -368,7 +378,7 @@ export async function runOrchestrator( // SDK-divergent steps ship per-framework variants, so resolve against // the menu with the session's framework before installing. const variantId = resolveSkillVariantId( - menuSkillIds, + menuSkillEntries, skillId, session.skillId, ); diff --git a/src/lib/agent/runner/switchboard/models.ts b/src/lib/agent/runner/switchboard/models.ts index b1a7383b4..e2dbdfe4b 100644 --- a/src/lib/agent/runner/switchboard/models.ts +++ b/src/lib/agent/runner/switchboard/models.ts @@ -27,13 +27,20 @@ import { import { RUN_SURFACE } from '@env'; /** Reasoning effort. pi maps it to `reasoning_effort` for openai-completions. */ -export type ThinkingLevel = - | 'off' - | 'minimal' - | 'low' - | 'medium' - | 'high' - | 'xhigh'; +const THINKING_LEVELS = [ + 'off', + 'minimal', + 'low', + 'medium', + 'high', + 'xhigh', +] as const; +export type ThinkingLevel = (typeof THINKING_LEVELS)[number]; + +/** Whether a value (e.g. remote prompt frontmatter) names a valid effort. */ +export function isThinkingLevel(value: unknown): value is ThinkingLevel { + return (THINKING_LEVELS as readonly unknown[]).includes(value); +} export interface ModelCapabilities { /** Model supports reasoning; safe to request reasoning effort. */ @@ -53,11 +60,12 @@ export const MODEL_CAPABILITIES: Record = { [GPT5_MODEL]: { reasoning: true, thinkingLevel: 'low' }, [GPT5_4_MODEL]: { reasoning: true, thinkingLevel: 'low' }, // Latest openai flagship line; all reasoning models, so they must opt in past - // the openai-completions default (reasoning off). Luna stays low for cheap, - // short-context mechanical work; terra runs medium as the sonnet-tier parallel - // — enough reasoning depth for the judgment tasks without high's latency blowup. + // the openai-completions default (reasoning off). Low effort keeps the + // linear run fast — this table tunes the LINEAR run only. Orchestrator + // agents own their effort per task via prompt frontmatter (`effort_pi`), + // which overrides the table without changing it. [GPT5_6_LUNA_MODEL]: { reasoning: true, thinkingLevel: 'low' }, - [GPT5_6_TERRA_MODEL]: { reasoning: true, thinkingLevel: 'medium' }, + [GPT5_6_TERRA_MODEL]: { reasoning: true, thinkingLevel: 'low' }, [GPT5_6_SOL_MODEL]: { reasoning: true, thinkingLevel: 'low' }, [GPT5_5_MODEL]: { reasoning: true, thinkingLevel: 'low' }, // The pi runner's paired model — a smaller openai reasoning model. Medium diff --git a/src/lib/wizard-tools.ts b/src/lib/wizard-tools.ts index b1efc0717..f77679bf2 100644 --- a/src/lib/wizard-tools.ts +++ b/src/lib/wizard-tools.ts @@ -46,7 +46,17 @@ async function getSDKModule(): Promise { // Skill types // --------------------------------------------------------------------------- -export type SkillEntry = { id: string; name: string; downloadUrl: string }; +export type SkillEntry = { + id: string; + name: string; + downloadUrl: string; + /** The hyphenated skill-group prefix of `id` (e.g. `posthog-integration-install`). */ + group?: string; + /** The detection id this variant serves (e.g. `rails`, `react-router`). */ + framework?: string; + /** The variant a bare framework id resolves to when its family has several. */ + default?: boolean; +}; /** * Entry in the wizard's runtime CLI registry. Mirrors the shape context-mill From 196f82422df939b4a6dbf73ad0226a6079720027 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Wed, 15 Jul 2026 11:34:09 -0400 Subject: [PATCH 2/8] chore(pi): trim review-fix comments to one-liners Generated-By: PostHog Code Task-Id: fafc230d-6f14-4e4d-9462-0e7f18a1eec1 --- .../__tests__/agent-prompt-loader.test.ts | 3 +- src/lib/agent/agent-prompt-loader.ts | 14 ++--- .../runner/__tests__/switchboard.test.ts | 7 +-- src/lib/agent/runner/harness/pi/index.ts | 16 +----- src/lib/agent/runner/harness/pi/shared.ts | 57 +++---------------- src/lib/agent/runner/harness/pi/task.ts | 9 +-- .../__tests__/variant-resolution.test.ts | 4 +- .../orchestrator/orchestrator-runner.ts | 15 +---- src/lib/agent/runner/switchboard/models.ts | 5 +- 9 files changed, 26 insertions(+), 104 deletions(-) diff --git a/src/lib/agent/__tests__/agent-prompt-loader.test.ts b/src/lib/agent/__tests__/agent-prompt-loader.test.ts index b25ed90bb..5ba580ffd 100644 --- a/src/lib/agent/__tests__/agent-prompt-loader.test.ts +++ b/src/lib/agent/__tests__/agent-prompt-loader.test.ts @@ -335,8 +335,7 @@ describe('taskModelSpec', () => { expect(taskModelSpec(registry, task as never, Harness.pi).model).toBe( 'prompt-model', ); - // An empty column stays undefined — the CALLER falls back to its - // switchboard pick, so a pi run degrades to the pi model, not sonnet. + // An empty column stays undefined — the caller falls back to its switchboard pick. expect( taskModelSpec(registry, task as never, Harness.anthropic).model, ).toBeUndefined(); diff --git a/src/lib/agent/agent-prompt-loader.ts b/src/lib/agent/agent-prompt-loader.ts index b6885fcb3..1658dc418 100644 --- a/src/lib/agent/agent-prompt-loader.ts +++ b/src/lib/agent/agent-prompt-loader.ts @@ -211,10 +211,8 @@ function toStringArray(value: unknown): string[] { * Parse the leading `---` frontmatter block and the markdown body. The * frontmatter is a small, known schema (scalars and inline `[a, b]` arrays), so * a tiny parser covers it without a YAML dependency. Inline `# comments` after a - * value are stripped. `fallbackType` is the menu id, used when the body omits - * `type:`; `fallbackFlow` is the menu entry's flow, used when the body omits - * `flow:` — so a prompt the menu placed in a flow is never silently dropped - * by the registry's flow filter. + * value are stripped. `fallbackType` (the menu id) and `fallbackFlow` (the + * menu entry's flow) apply when the frontmatter omits `type:`/`flow:`. */ export function parseAgentPrompt( text: string, @@ -244,8 +242,7 @@ export function parseAgentPrompt( } const str = (v: unknown) => (typeof v === 'string' ? v : undefined); - // Effort is remote data; a typo must not ride into a session as a bogus - // reasoning level. Validate here so downstream code carries ThinkingLevel. + // Effort is remote data — reject typos here so downstream carries ThinkingLevel. const effort = (v: unknown, key: string): ThinkingLevel | undefined => { if (v === undefined) return undefined; if (isThinkingLevel(v)) return v; @@ -400,10 +397,7 @@ export function resolveTask( }; } -/** The model + effort a task runs on for a harness: enqueue override, then the - * prompt's per-profile frontmatter. No default is baked in here — the caller - * falls back to its switchboard pick, so each harness degrades to its own - * column's model rather than everything collapsing to the anthropic default. */ +/** Enqueue override, then per-profile frontmatter; no default — the caller falls back to its switchboard pick. */ export function taskModelSpec( registry: AgentRegistry, task: QueuedTask, diff --git a/src/lib/agent/runner/__tests__/switchboard.test.ts b/src/lib/agent/runner/__tests__/switchboard.test.ts index 82d6fa7c6..e5c68d670 100644 --- a/src/lib/agent/runner/__tests__/switchboard.test.ts +++ b/src/lib/agent/runner/__tests__/switchboard.test.ts @@ -328,8 +328,7 @@ describe('switchboard modelCapabilities', () => { expect(modelCapabilities(GPT5_MODEL).thinkingLevel).toBe('low'); expect(modelCapabilities(GPT5_MINI_MODEL).thinkingLevel).toBe('medium'); // The gpt-5.6 line + gpt-5.5 are reasoning models despite the openai/ prefix; they opt in past the default-off. - // All low in the table — the table tunes the LINEAR run; orchestrator - // agents raise effort per task via their prompt frontmatter instead. + // All low — the table tunes the linear run; orchestrator agents raise effort via frontmatter. for (const m of [ GPT5_6_LUNA_MODEL, GPT5_6_TERRA_MODEL, @@ -404,9 +403,7 @@ describe('switchboard wizard-pi-effort flag', () => { }); it('opts out with applyEffortFlag:false — orchestrator tasks keep the table effort', () => { - // The flag is a linear-run knob; a per-task agent ignores it and keeps the - // table level (its own effort comes from prompt frontmatter instead), - // even with the flag set to high. + // The flag is a linear-run knob; a per-task agent ignores it even when set to high. expect( modelCapabilities( GPT5_6_TERRA_MODEL, diff --git a/src/lib/agent/runner/harness/pi/index.ts b/src/lib/agent/runner/harness/pi/index.ts index 481fd4a87..11c0cfb4e 100644 --- a/src/lib/agent/runner/harness/pi/index.ts +++ b/src/lib/agent/runner/harness/pi/index.ts @@ -139,9 +139,6 @@ export const piBackend: AgentHarness = { const sdk = await import('@earendil-works/pi-coding-agent'); const { getAgentDir } = sdk; - // The gateway provider spec + registry wiring is shared with the - // orchestrator's per-task sessions (shared.ts / gateway.ts). Linear runs - // honour the wizard-pi-effort flag (the default). const gateway = resolveGatewayModel(sdk, boot, modelId); if (!gateway) { return { @@ -180,10 +177,7 @@ export const piBackend: AgentHarness = { const { prewarmYaraScanner } = await import('@lib/yara-hooks'); void prewarmYaraScanner(); - // Wire the real PostHog MCP into pi (#10), best-effort — if it can't - // load or connect, the run continues (minus the dashboard step) rather - // than failing the whole integration. The security factory is always - // first. + // Real PostHog MCP (#10), best-effort; the security factory is always first. const extensionFactories = [security.factory] as Array< (pi: unknown) => void >; @@ -202,10 +196,7 @@ export const piBackend: AgentHarness = { const { createDispatchAgentTool } = await import('./subagent'); // Created once so the run loop can read the store for the completion guard. const wizardTaskTools = createWizardPiTaskTools(); - // Built-ins re-registered explicitly (`noTools: 'builtin'` in the shared - // session builder disables pi's defaults): reads/searches parallel, - // edit/write/bash sequential, bash on the scrubbed env. The one bash is - // shared with the subagent so the lockdown is inherited. + // The one env-scrubbed bash is shared with the subagent so the lockdown is inherited. const coding = piCodingToolFactories(sdk, session.installDir); const scrubbedBash = coding.bash(); @@ -259,8 +250,7 @@ export const piBackend: AgentHarness = { customTools, }); - // Map pi events onto the run spinner + the log file; counts drive the - // no-progress guard below. + // counts drive the no-progress guard below. const { counts, unsubscribe } = watchSession(agentSession, { tag: '[pi]', spinner, diff --git a/src/lib/agent/runner/harness/pi/shared.ts b/src/lib/agent/runner/harness/pi/shared.ts index 03bbe90fd..0f6e9b690 100644 --- a/src/lib/agent/runner/harness/pi/shared.ts +++ b/src/lib/agent/runner/harness/pi/shared.ts @@ -1,14 +1,7 @@ /** - * The pi session machinery shared by the harness's two entry points: the - * linear `run()` (index.ts) and the orchestrator's per-task `runTask()` - * (task.ts). One copy of the gateway/registry wiring, the hermetic resource - * loader, the coding-tool definitions, the session event handler, and the - * error/usage bookkeeping — so the two runs configure the same machinery - * instead of drifting apart (the 429 sniff and the analytics shape especially). - * - * No typebox in this module graph: index.ts imports it statically (the - * CommonJS unit-test seam loads it), while the SDK itself always arrives as - * the caller's lazy import, passed in as `sdk`. + * pi session machinery shared by the linear run (index.ts) and the + * orchestrator's per-task runs (task.ts). No typebox in this module graph — + * the SDK always arrives as the caller's lazy import, passed in as `sdk`. */ import { getUI, type SpinnerHandle } from '@ui'; @@ -137,13 +130,7 @@ export function lastStatusLine(textBlock: string): string | undefined { return status || undefined; } -/** - * Register the PostHog gateway on a fresh in-memory registry and resolve the - * model. Undefined when the model can't be resolved — the caller returns its - * API_ERROR. `applyEffortFlag`/`effort` follow `buildGatewayProvider`: - * the linear run honours the wizard-pi-effort flag, a per-task run passes - * `applyEffortFlag: false` and its own frontmatter effort. - */ +/** Register the PostHog gateway on a fresh in-memory registry and resolve the model; undefined when it can't be. */ export function resolveGatewayModel( sdk: PiSdk, boot: BootstrapResult, @@ -173,12 +160,7 @@ export function resolveGatewayModel( return { registry, model, caps, gatewayUrl }; } -/** - * Wire the real PostHog MCP, best-effort: if the adapter can't load or - * connect, the run continues (minus the posthog_* tools) rather than failing. - * The caller pushes `extensionFactory` after the security factory and calls - * `cleanup` when the session ends. - */ +/** Wire the real PostHog MCP, best-effort — on failure the run continues without the posthog_* tools. */ export async function connectPostHogMcp( sdk: PiSdk, boot: BootstrapResult, @@ -202,11 +184,7 @@ export async function connectPostHogMcp( } } -/** - * The stock coding tools rooted at `dir`, each tagged with its execution mode - * and bash wired to the scrubbed subprocess env. Factories, not instances, so - * a caller registers exactly the tools its allow list grants. - */ +/** The stock coding tools rooted at `dir` (modes tagged, bash env-scrubbed), as factories so callers register only what their allow list grants. */ export function piCodingToolFactories(sdk: PiSdk, dir: string) { return { read: () => withMode(sdk.createReadToolDefinition(dir), 'parallel'), @@ -225,13 +203,7 @@ export function piCodingToolFactories(sdk: PiSdk, dir: string) { } as const; } -/** - * A hermetic pi session: the given system prompt and extensions only — no - * disk-discovered extensions, skills, context files, prompt templates, or - * themes from the target project — with pi's default built-in tools disabled - * so `customTools` is the entire tool surface. Fires the extension lifecycle - * (`bindExtensions`) before returning, which the MCP adapter connects on. - */ +/** A hermetic pi session: nothing loads from the target project, `customTools` is the entire tool surface, extensions are bound before returning. */ export async function createHermeticSession( sdk: PiSdk, opts: { @@ -277,13 +249,7 @@ export interface SessionCounts { toolCalls: number; } -/** - * Map pi session events onto the run spinner + the log file, mirroring the - * anthropic path's log shape (assistant turns + tool I/O) and driving the - * single run spinner with one stable status at a time. `[STATUS]` lines - * surface into the live spinner and status history; outro markers apply as - * they stream; every assistant line feeds `signals`. - */ +/** Map pi session events onto the spinner, status history, signals, and log, keeping the counters live. */ export function watchSession( agentSession: PiAgentSession, opts: { tag: string; spinner: SpinnerHandle; signals: AgentOutputSignals }, @@ -367,12 +333,7 @@ export function classifyRunError(err: unknown): { return { error, message }; } -/** - * The `agent completed` capture both entry points send, plus one parseable - * usage line in the log so a run's per-unit time and cost are observable - * without analytics access. `task=` appears only when the run carries a - * task_type (orchestrator units); the linear run has none. - */ +/** The `agent completed` capture plus one parseable usage log line (`task=` only when a task_type rides along). */ export function captureAgentUsage(opts: { tag: string; modelId: string; diff --git a/src/lib/agent/runner/harness/pi/task.ts b/src/lib/agent/runner/harness/pi/task.ts index e17a25719..a2bd1ad0d 100644 --- a/src/lib/agent/runner/harness/pi/task.ts +++ b/src/lib/agent/runner/harness/pi/task.ts @@ -1,11 +1,8 @@ /** * Orchestrator-mode execution on pi: one fresh pi session per unit of work — - * the seed plan, or one drained task. The session machinery (gateway model, - * hermetic loader, event watching, error/usage bookkeeping) is `shared.ts`, - * common with the linear run; the linear pipeline's own concerns (skill menu, - * todo panel, event-plan cleanup) stay in `index.ts`. This module configures - * the leaner per-task session: the task's allowed coding tools, the wizard env - * tools, and the in-process orchestrator queue tools. + * the seed plan, or one drained task. The session machinery lives in + * `shared.ts`; this module configures the leaner per-task session: the task's + * allowed coding tools, the wizard env tools, and the queue tools. * * The task's `allowedTools` / `disallowedTools` arrive in the wizard's tool * vocabulary (`Read`, `Edit`, `Glob`, …, plus MCP-qualified orchestrator names diff --git a/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts b/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts index 9367bed4d..2aee5044f 100644 --- a/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts +++ b/src/lib/agent/runner/sequence/orchestrator/__tests__/variant-resolution.test.ts @@ -3,9 +3,7 @@ import { resolveSkillVariantId } from '../orchestrator-runner'; import { Integration } from '@lib/constants'; import type { SkillEntry } from '@lib/wizard-tools'; -// The integration group exactly as the context-mill build emits it into -// skill-menu.json (id + group + framework + default), so this suite pins the -// real cross-repo contract instead of a fixture that mirrors the resolver. +// Pinned from the real built skill-menu.json, so this suite tests the actual cross-repo contract. const INTEGRATION_ENTRIES = [ { id: 'integration-nextjs-app-router', framework: 'nextjs', default: true }, { id: 'integration-nextjs-pages-router', framework: 'nextjs' }, diff --git a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts index a7e7ed3e7..6319e1638 100644 --- a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -92,15 +92,7 @@ async function fetchSkillMenuEntries( return Object.values(menu.categories).flat(); } -/** - * Resolve a bare skill id + the session's framework to the menu id, from the - * `group`/`framework`/`default` fields the menu declares: a full menu id (or - * single-variant skill) resolves to itself; otherwise the entry whose group is - * the bare id and whose framework matches — the family's marked default when - * several variants serve one framework (e.g. app vs pages router). No wizard- - * side vocabulary: context-mill owns which variant serves which framework, so - * a rename there cannot silently strand the resolution here. - */ +/** Menu id for a bare skill id + framework via the menu's declared group/framework/default fields; undefined when nothing matches. */ export function resolveSkillVariantId( entries: readonly SkillEntry[], skillId: string, @@ -255,10 +247,7 @@ export async function runOrchestrator( ); } - // Preflight the HOW: resolve every task's mini-skills against the menu now, - // so a variant gap between the registry and the menu surfaces (log + - // analytics) before any agent runs — not as a silently skill-less task - // discovered mid-drain, whose zero-diff run would look like a success. + // Preflight every task's mini-skills so a missing variant surfaces before any agent runs. for (const type of registry.types) { for (const skillId of registry.get(type)?.skills ?? []) { if (!resolveSkillVariantId(menuSkillEntries, skillId, session.skillId)) { diff --git a/src/lib/agent/runner/switchboard/models.ts b/src/lib/agent/runner/switchboard/models.ts index e2dbdfe4b..4a6c726c5 100644 --- a/src/lib/agent/runner/switchboard/models.ts +++ b/src/lib/agent/runner/switchboard/models.ts @@ -60,10 +60,7 @@ export const MODEL_CAPABILITIES: Record = { [GPT5_MODEL]: { reasoning: true, thinkingLevel: 'low' }, [GPT5_4_MODEL]: { reasoning: true, thinkingLevel: 'low' }, // Latest openai flagship line; all reasoning models, so they must opt in past - // the openai-completions default (reasoning off). Low effort keeps the - // linear run fast — this table tunes the LINEAR run only. Orchestrator - // agents own their effort per task via prompt frontmatter (`effort_pi`), - // which overrides the table without changing it. + // the openai-completions default (reasoning off). Low effort keeps a run fast. [GPT5_6_LUNA_MODEL]: { reasoning: true, thinkingLevel: 'low' }, [GPT5_6_TERRA_MODEL]: { reasoning: true, thinkingLevel: 'low' }, [GPT5_6_SOL_MODEL]: { reasoning: true, thinkingLevel: 'low' }, From ef240196acd770d3dee3922fa5d50305a6f1dfc1 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Wed, 15 Jul 2026 12:33:54 -0400 Subject: [PATCH 3/8] revert(pi): restore terra's medium table effort The medium row is terra's default for any run that doesn't name an effort itself; the "linear isolation" concern was hypothetical (linear doesn't run terra) and the revert only degraded the real default. models.ts and switchboard.test.ts terra lines are back to the base branch verbatim. Generated-By: PostHog Code Task-Id: fafc230d-6f14-4e4d-9462-0e7f18a1eec1 --- src/lib/agent/runner/__tests__/switchboard.test.ts | 12 ++++++++---- src/lib/agent/runner/switchboard/models.ts | 6 ++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/lib/agent/runner/__tests__/switchboard.test.ts b/src/lib/agent/runner/__tests__/switchboard.test.ts index e5c68d670..41051092f 100644 --- a/src/lib/agent/runner/__tests__/switchboard.test.ts +++ b/src/lib/agent/runner/__tests__/switchboard.test.ts @@ -328,7 +328,6 @@ describe('switchboard modelCapabilities', () => { expect(modelCapabilities(GPT5_MODEL).thinkingLevel).toBe('low'); expect(modelCapabilities(GPT5_MINI_MODEL).thinkingLevel).toBe('medium'); // The gpt-5.6 line + gpt-5.5 are reasoning models despite the openai/ prefix; they opt in past the default-off. - // All low — the table tunes the linear run; orchestrator agents raise effort via frontmatter. for (const m of [ GPT5_6_LUNA_MODEL, GPT5_6_TERRA_MODEL, @@ -336,8 +335,12 @@ describe('switchboard modelCapabilities', () => { GPT5_5_MODEL, ]) { expect(modelCapabilities(m).reasoning).toBe(true); - expect(modelCapabilities(m).thinkingLevel).toBe('low'); } + // luna/sol/5.5 stay low (fast); terra runs medium as the sonnet-tier parallel. + expect(modelCapabilities(GPT5_6_LUNA_MODEL).thinkingLevel).toBe('low'); + expect(modelCapabilities(GPT5_6_TERRA_MODEL).thinkingLevel).toBe('medium'); + expect(modelCapabilities(GPT5_6_SOL_MODEL).thinkingLevel).toBe('low'); + expect(modelCapabilities(GPT5_5_MODEL).thinkingLevel).toBe('low'); // Anthropic default carries no explicit effort — the harness default stands. expect( modelCapabilities(DEFAULT_AGENT_MODEL).thinkingLevel, @@ -403,14 +406,15 @@ describe('switchboard wizard-pi-effort flag', () => { }); it('opts out with applyEffortFlag:false — orchestrator tasks keep the table effort', () => { - // The flag is a linear-run knob; a per-task agent ignores it even when set to high. + // The flag is a linear-run knob; a per-task agent ignores it and keeps its + // own tuned level (terra medium), even with the flag set to high. expect( modelCapabilities( GPT5_6_TERRA_MODEL, { ...PI_ON, [WIZARD_PI_EFFORT_FLAG_KEY]: 'high' }, { applyEffortFlag: false }, ).thinkingLevel, - ).toBe('low'); + ).toBe('medium'); expect( modelCapabilities( GPT5_6_LUNA_MODEL, diff --git a/src/lib/agent/runner/switchboard/models.ts b/src/lib/agent/runner/switchboard/models.ts index 4a6c726c5..494baab69 100644 --- a/src/lib/agent/runner/switchboard/models.ts +++ b/src/lib/agent/runner/switchboard/models.ts @@ -60,9 +60,11 @@ export const MODEL_CAPABILITIES: Record = { [GPT5_MODEL]: { reasoning: true, thinkingLevel: 'low' }, [GPT5_4_MODEL]: { reasoning: true, thinkingLevel: 'low' }, // Latest openai flagship line; all reasoning models, so they must opt in past - // the openai-completions default (reasoning off). Low effort keeps a run fast. + // the openai-completions default (reasoning off). Luna stays low for cheap, + // short-context mechanical work; terra runs medium as the sonnet-tier parallel + // — enough reasoning depth for the judgment tasks without high's latency blowup. [GPT5_6_LUNA_MODEL]: { reasoning: true, thinkingLevel: 'low' }, - [GPT5_6_TERRA_MODEL]: { reasoning: true, thinkingLevel: 'low' }, + [GPT5_6_TERRA_MODEL]: { reasoning: true, thinkingLevel: 'medium' }, [GPT5_6_SOL_MODEL]: { reasoning: true, thinkingLevel: 'low' }, [GPT5_5_MODEL]: { reasoning: true, thinkingLevel: 'low' }, // The pi runner's paired model — a smaller openai reasoning model. Medium From 5d1c2eef0e42125ff5664be9e417772dffd0faf6 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Wed, 15 Jul 2026 12:36:33 -0400 Subject: [PATCH 4/8] =?UTF-8?q?revert(pi):=20drop=20the=20shared.ts=20extr?= =?UTF-8?q?action=20=E2=80=94=20runners=20stay=20self-contained?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The switchboard is the single decision point; a shared layer between it and the runners is a place decisions could creep into. index.ts, task.ts, and subagent.ts return to the base branch verbatim; the typed effort in gateway.ts/types.ts stays. Generated-By: PostHog Code Task-Id: fafc230d-6f14-4e4d-9462-0e7f18a1eec1 --- .../harness/pi/__tests__/env-lockdown.test.ts | 2 +- .../harness/pi/__tests__/status-line.test.ts | 2 +- src/lib/agent/runner/harness/pi/index.ts | 364 ++++++++++++++--- src/lib/agent/runner/harness/pi/shared.ts | 369 ------------------ src/lib/agent/runner/harness/pi/subagent.ts | 16 +- src/lib/agent/runner/harness/pi/task.ts | 219 ++++++++--- 6 files changed, 493 insertions(+), 479 deletions(-) delete mode 100644 src/lib/agent/runner/harness/pi/shared.ts diff --git a/src/lib/agent/runner/harness/pi/__tests__/env-lockdown.test.ts b/src/lib/agent/runner/harness/pi/__tests__/env-lockdown.test.ts index e6600e234..1034e0bca 100644 --- a/src/lib/agent/runner/harness/pi/__tests__/env-lockdown.test.ts +++ b/src/lib/agent/runner/harness/pi/__tests__/env-lockdown.test.ts @@ -4,7 +4,7 @@ * drops everything else — the leak that exposed the test key before. */ -import { buildScrubbedEnv } from '../shared'; +import { buildScrubbedEnv } from '..'; describe('buildScrubbedEnv', () => { const saved = { ...process.env }; diff --git a/src/lib/agent/runner/harness/pi/__tests__/status-line.test.ts b/src/lib/agent/runner/harness/pi/__tests__/status-line.test.ts index 5f92a763a..b2920ab18 100644 --- a/src/lib/agent/runner/harness/pi/__tests__/status-line.test.ts +++ b/src/lib/agent/runner/harness/pi/__tests__/status-line.test.ts @@ -6,7 +6,7 @@ import { describe, it, expect } from 'vitest'; import { AgentSignals } from '@lib/agent/signals'; -import { lastStatusLine } from '../shared'; +import { lastStatusLine } from '..'; const S = AgentSignals.STATUS; // '[STATUS]' diff --git a/src/lib/agent/runner/harness/pi/index.ts b/src/lib/agent/runner/harness/pi/index.ts index 11c0cfb4e..16d815f27 100644 --- a/src/lib/agent/runner/harness/pi/index.ts +++ b/src/lib/agent/runner/harness/pi/index.ts @@ -16,22 +16,17 @@ import fs from 'fs'; import path from 'path'; import { getUI } from '@ui'; import { getLogFilePath, logToFile } from '@utils/debug'; -import { Harness, WIZARD_REMARK_EVENT_NAME } from '@lib/constants'; +import { + Harness, + WIZARD_REMARK_EVENT_NAME, + WIZARD_USER_AGENT, +} from '@lib/constants'; import { analytics } from '@utils/analytics'; import { AgentErrorType } from '@lib/agent/agent-interface'; -import { REMARK_INSTRUCTION } from '@lib/agent/signals'; +import { AgentSignals, REMARK_INSTRUCTION } from '@lib/agent/signals'; import { AgentOutputSignals } from '@lib/agent/output-signals'; import { getWizardCommandments } from '@lib/agent/commandments'; -import { - captureAgentUsage, - classifyRunError, - connectPostHogMcp, - createHermeticSession, - piCodingToolFactories, - resolveGatewayModel, - startRunClock, - watchSession, -} from './shared'; +import { buildGatewayProvider, GATEWAY_PROVIDER } from './gateway'; import type { AgentResult, AgentHarness, @@ -99,6 +94,113 @@ function piMcpContext(boot: BootstrapResult, instructions?: string): string { ].join('\n'); } +/** + * The ONLY environment variables pi's tool subprocesses (bash → npm/pip/…) are + * allowed to see. Everything else — every secret (POSTHOG_PERSONAL_API_KEY, + * ANTHROPIC_*, AWS_*), every ambient credential, the parent process's whole env + * — is dropped before a child is spawned. pi's own gateway auth is programmatic + * (the access token never lives in env), so a minimal env costs the agent + * nothing while closing the leak that exposed the key before. Kept to what a + * package manager genuinely needs to run. + */ +const ALLOWED_SUBPROCESS_ENV_KEYS = [ + 'PATH', + 'HOME', + 'SHELL', + 'USER', + 'LOGNAME', + 'TMPDIR', + 'TMP', + 'TEMP', + 'TERM', + 'LANG', + 'LC_ALL', + 'LC_CTYPE', + 'NODE_EXTRA_CA_CERTS', + 'SSL_CERT_FILE', + 'SSL_CERT_DIR', + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'NO_PROXY', + 'http_proxy', + 'https_proxy', + 'no_proxy', +]; + +/** A fresh subprocess env holding only the allowlisted keys present in process.env. */ +export function buildScrubbedEnv(): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const key of ALLOWED_SUBPROCESS_ENV_KEYS) { + const value = process.env[key]; + if (value !== undefined) env[key] = value; + } + return env; +} + +/** + * Tag a tool with an execution mode (mutates + returns it). Read-only tools are + * `parallel` so a single turn that batches independent reads/searches runs them + * at once; mutating/install tools are `sequential` so a batch never races writes + * or concurrent installs. pi-agent-core runs a batch in parallel only when no + * tool in it is `sequential`. + */ +export function withMode(tool: T, mode: 'sequential' | 'parallel'): T { + (tool as { executionMode?: 'sequential' | 'parallel' }).executionMode = mode; + return tool; +} + +/** Pull plain text out of a pi AgentMessage (content is text/image blocks). */ +export function extractText(message: unknown): string { + const content = (message as { content?: unknown })?.content; + if (typeof content === 'string') return content; + if (Array.isArray(content)) { + return content + .filter((c): c is { type: string; text: string } => { + const block = c as { type?: string; text?: unknown }; + return block?.type === 'text' && typeof block.text === 'string'; + }) + .map((c) => c.text) + .join(''); + } + return ''; +} + +/** + * Surface `[DASHBOARD_URL]` / `[NOTEBOOK_URL]` markers the agent prints (after + * the MCP creates them) into the outro link, mirroring the anthropic path's + * signal parsing (#9). The marker carries the URL the MCP returned. + */ +export function applyOutroMarkers(textBlock: string): void { + const markers: Array<[string, (url: string) => void]> = [ + [AgentSignals.DASHBOARD_URL, (url) => getUI().setDashboardUrl(url)], + [AgentSignals.NOTEBOOK_URL, (url) => getUI().setNotebookUrl(url)], + ]; + for (const [marker, apply] of markers) { + const idx = textBlock.indexOf(marker); + if (idx === -1) continue; + const url = textBlock + .slice(idx + marker.length) + .trim() + .split(/\s/)[0]; + if (url) apply(url); + } +} + +/** + * The text of the last `[STATUS] …` line in a block, if any. Last wins so the + * spinner shows the most recent action when a turn prints several. + */ +export function lastStatusLine(textBlock: string): string | undefined { + let status: string | undefined; + for (const line of textBlock.split('\n')) { + const idx = line.indexOf(AgentSignals.STATUS); + if (idx !== -1) { + status = line.slice(idx + AgentSignals.STATUS.length).trim(); + } + } + return status || undefined; +} + /** Cap on completion-guard re-prompts while tasks remain open (see the run loop). */ const MAX_CONTINUE_NUDGES = 20; @@ -127,8 +229,19 @@ export const piBackend: AgentHarness = { spinner.start(config.spinnerMessage ?? 'Customizing your PostHog setup...'); // Same `agent completed`/`agent aborted` shape as anthropic. + const startTime = Date.now(); const signals = new AgentOutputSignals(); - const runDurations = startRunClock(); + let assistantTurns = 0; + // Tool calls across the whole run. Zero means the agent only ever produced + // text and never acted — a no-op that leaves the project untouched. + let toolCalls = 0; + const runDurations = () => { + const durationMs = Date.now() - startTime; + return { + duration_ms: durationMs, + duration_seconds: Math.round(durationMs / 1000), + }; + }; const captureAborted = () => analytics.wizardCapture('agent aborted', { ...runDurations(), @@ -136,17 +249,41 @@ export const piBackend: AgentHarness = { }); try { - const sdk = await import('@earendil-works/pi-coding-agent'); - const { getAgentDir } = sdk; + const { + createAgentSession, + DefaultResourceLoader, + SessionManager, + AuthStorage, + ModelRegistry, + getAgentDir, + createLsToolDefinition, + createFindToolDefinition, + createGrepToolDefinition, + createBashToolDefinition, + createReadToolDefinition, + createEditToolDefinition, + createWriteToolDefinition, + } = await import('@earendil-works/pi-coding-agent'); + + // the claude-agent-sdk path. The provider spec is shared with the + // orchestrator's per-task sessions (gateway.ts). + const { provider, caps, gatewayUrl } = buildGatewayProvider({ + gatewayUrl: boot.credentials.host.gatewayUrl, + accessToken: boot.credentials.accessToken, + wizardMetadata: boot.wizardMetadata, + wizardFlags: boot.wizardFlags, + modelId, + }); + const registry = ModelRegistry.inMemory(AuthStorage.create()); + registry.registerProvider(GATEWAY_PROVIDER, provider as never); - const gateway = resolveGatewayModel(sdk, boot, modelId); - if (!gateway) { + const model = registry.find(GATEWAY_PROVIDER, modelId); + if (!model) { return { error: AgentErrorType.API_ERROR, message: 'pi: gateway model could not be resolved', }; } - const { registry, model, caps, gatewayUrl } = gateway; // System prompt = wizard commandments. Skip project context files / // user extensions / skills so the run is hermetic; skills discovery is a @@ -177,13 +314,48 @@ export const piBackend: AgentHarness = { const { prewarmYaraScanner } = await import('@lib/yara-hooks'); void prewarmYaraScanner(); - // Real PostHog MCP (#10), best-effort; the security factory is always first. + // Wire the real PostHog MCP into pi (#10): load pi's MCP adapter and point + // it at the hosted MCP the anthropic path uses, so dashboards/insights are + // created through the sanctioned MCP. Best-effort — if it can't load or + // connect, the run continues (minus the dashboard step) rather than failing + // the whole integration. The security factory is always first. const extensionFactories = [security.factory] as Array< (pi: unknown) => void >; - const mcp = await connectPostHogMcp(sdk, boot, '[pi]'); - if (mcp.extensionFactory) extensionFactories.push(mcp.extensionFactory); - const mcpCleanup = mcp.cleanup; + let mcpCleanup: (() => void) | undefined; + let mcpInstructions: string | undefined; + try { + const { setupPostHogMcp } = await import('./mcp'); + const mcp = await setupPostHogMcp({ + agentDir: getAgentDir(), + mcpUrl: boot.credentials.host.mcpUrl, + accessToken: boot.credentials.accessToken, + userAgent: WIZARD_USER_AGENT, + }); + extensionFactories.push(mcp.extensionFactory); + mcpCleanup = mcp.cleanup; + mcpInstructions = mcp.instructions; + } catch (err) { + logToFile(`[pi] PostHog MCP setup skipped: ${String(err)}`); + } + + const resourceLoader = new DefaultResourceLoader({ + cwd: session.installDir, + agentDir: getAgentDir(), + systemPrompt: + getWizardCommandments() + + '\n' + + PI_RUNTIME_NOTES + + '\n' + + piMcpContext(boot, mcpInstructions), + noExtensions: true, + noSkills: true, + noContextFiles: true, + noPromptTemplates: true, + noThemes: true, + extensionFactories, + }); + await resourceLoader.reload(); // Wizard capabilities as custom tools (pi has no MCP): skill // discovery/install + fenced .env edits, same names as the MCP server so @@ -196,21 +368,31 @@ export const piBackend: AgentHarness = { const { createDispatchAgentTool } = await import('./subagent'); // Created once so the run loop can read the store for the completion guard. const wizardTaskTools = createWizardPiTaskTools(); - // The one env-scrubbed bash is shared with the subagent so the lockdown is inherited. - const coding = piCodingToolFactories(sdk, session.installDir); - const scrubbedBash = coding.bash(); + // The one bash the agent (and its subagents) may use: every subprocess it + // spawns gets a scrubbed env, so no secret or ambient variable reaches an + // `npm install`. Shared with the subagent so the lockdown is inherited. + const scrubbedBash = withMode( + createBashToolDefinition(session.installDir, { + spawnHook: (ctx) => ({ ...ctx, env: buildScrubbedEnv() }), + }), + 'sequential', + ); const customTools = [ - coding.read(), - coding.edit(), - coding.write(), + // Built-ins re-registered explicitly. `noTools: 'builtin'` disables pi's + // defaults so we can supply the env-scrubbed bash above; read/edit/write + // are the stock definitions. Reads run in parallel so a batched turn of + // independent reads executes at once; edit/write/bash stay sequential. + withMode(createReadToolDefinition(session.installDir), 'parallel'), + withMode(createEditToolDefinition(session.installDir), 'sequential'), + withMode(createWriteToolDefinition(session.installDir), 'sequential'), scrubbedBash, // Native ls/find/grep so the agent explores with proper tools instead // of fence-blocked `bash {ls/find}` (the profiled retry-spirals came // from this gap). Parallel — exploration batches cleanly. - coding.ls(), - coding.find(), - coding.grep(), + withMode(createLsToolDefinition(session.installDir), 'parallel'), + withMode(createFindToolDefinition(session.installDir), 'parallel'), + withMode(createGrepToolDefinition(session.installDir), 'parallel'), ...createWizardPiTools({ workingDirectory: session.installDir, skillsBaseUrl: boot.skillsBaseUrl, @@ -229,32 +411,85 @@ export const piBackend: AgentHarness = { agentDir: getAgentDir(), securityFactory: security.factory as (pi: unknown) => void, bashTool: scrubbedBash, - sdk, + sdk: { createAgentSession, DefaultResourceLoader, SessionManager }, }), ]; - const agentSession = await createHermeticSession(sdk, { - cwd: session.installDir, - systemPrompt: - getWizardCommandments() + - '\n' + - PI_RUNTIME_NOTES + - '\n' + - piMcpContext(boot, mcp.instructions), - extensionFactories, + const { session: agentSession } = await createAgentSession({ model, - registry, + modelRegistry: registry, // Reasoning effort from the switchboard capability matrix (undefined = // pi's default). Sent as `reasoning_effort` for openai-completions. thinkingLevel: caps.thinkingLevel, + cwd: session.installDir, + sessionManager: SessionManager.inMemory(session.installDir), + resourceLoader, + // Disable the default built-in tools; `customTools` re-registers + // read/edit/write + an env-scrubbed bash, so no subprocess inherits the + // host env. Custom + extension tools stay enabled. + noTools: 'builtin', customTools, }); - // counts drive the no-progress guard below. - const { counts, unsubscribe } = watchSession(agentSession, { - tag: '[pi]', - spinner, - signals, + // Fire the extension lifecycle — what interactive mode does via + // rebindCurrentSession. createAgentSession builds the session but does not + // emit session_start on its own, and the MCP adapter connects on that + // event; without this its tools report "MCP not initialized". + await agentSession.bindExtensions({}); + + // Map pi events onto the run spinner + the log file, mirroring the + // anthropic path's log shape (assistant turns + tool I/O) and driving the + // single run spinner with one stable status at a time (no overlap). + const unsubscribe = agentSession.subscribe((event) => { + switch (event.type) { + case 'message_end': { + // User prompts also emit message_end; only assistant turns count. + if ((event.message as { role?: string })?.role !== 'assistant') { + break; + } + assistantTurns += 1; + const assistant = extractText(event.message).trim(); + if (assistant) { + logToFile(`[pi] assistant: ${assistant.slice(0, 1000)}`); + applyOutroMarkers(assistant); + // Surface [STATUS] lines into the live spinner + status history, + // mirroring the anthropic path — pi otherwise drops them. + const statusText = lastStatusLine(assistant); + if (statusText) { + getUI().pushStatus(statusText); + spinner.message(statusText); + } + for (const line of assistant.split('\n')) signals.push(line); + } + break; + } + case 'tool_execution_start': { + toolCalls += 1; + const args = JSON.stringify(event.args ?? {}).slice(0, 200); + logToFile(`[pi] → ${event.toolName} ${args}`); + // Don't surface raw tool names in the spinner — the anthropic path + // doesn't, and it reads as noise. The Task panel (syncTodos) is the + // visible progress, matching the anthropic presentation. + break; + } + case 'tool_execution_end': { + if (event.isError) { + logToFile( + `[pi] ✗ ${event.toolName}: ${String(event.result).slice( + 0, + 300, + )}`, + ); + } + break; + } + case 'agent_end': { + logToFile(`[pi] agent_end (willRetry=${String(event.willRetry)})`); + break; + } + default: + break; + } }); try { @@ -305,17 +540,14 @@ export const piBackend: AgentHarness = { // pi ends a run on any tool-call-less turn, so guard against a hollow // success reaching the outro (nothing done, or stopped mid-plan). const openTasks = hasOpenTasks(wizardTaskTools.store); - const failure = completionFailure({ - toolCalls: counts.toolCalls, - openTasks, - }); + const failure = completionFailure({ toolCalls, openTasks }); if (failure === AgentErrorType.NO_PROGRESS) { spinner.stop('Agent made no changes'); logToFile( - `[pi] no progress: ${counts.assistantTurns} assistant turn(s), 0 tool calls`, + `[pi] no progress: ${assistantTurns} assistant turn(s), 0 tool calls`, ); analytics.wizardCapture('agent no progress', { - assistant_turns: counts.assistantTurns, + assistant_turns: assistantTurns, }); captureAborted(); return { error: failure }; @@ -352,22 +584,32 @@ export const piBackend: AgentHarness = { logToFile(`[pi] .posthog-events.json cleanup skipped: ${String(err)}`); } - captureAgentUsage({ - tag: '[pi]', - modelId, - counts, - durations: runDurations(), - stats: agentSession.getSessionStats(), + const stats = agentSession.getSessionStats(); + analytics.wizardCapture('agent completed', { + ...runDurations(), + model: modelId, + num_turns: assistantTurns, + // API-reported tokens only; no total_cost_usd — the API returns no + // cost, and $ai_generation already prices the run authoritatively. + input_tokens: stats.tokens.input, + output_tokens: stats.tokens.output, + cache_creation_input_tokens: stats.tokens.cacheWrite, + cache_read_input_tokens: stats.tokens.cacheRead, }); spinner.stop(config.successMessage ?? 'PostHog integration complete'); return {}; } catch (err) { - const { error, message } = classifyRunError(err); + const message = err instanceof Error ? err.message : String(err); logToFile(`[pi] run error: ${message}`); spinner.stop(config.errorMessage ?? `${config.integrationLabel} failed`); getUI().log.error(`pi backend error: ${message}`); captureAborted(); - return { error, message }; + + const lower = message.toLowerCase(); + if (lower.includes('rate limit') || lower.includes('429')) { + return { error: AgentErrorType.RATE_LIMIT, message }; + } + return { error: AgentErrorType.API_ERROR, message }; } }, diff --git a/src/lib/agent/runner/harness/pi/shared.ts b/src/lib/agent/runner/harness/pi/shared.ts deleted file mode 100644 index 0f6e9b690..000000000 --- a/src/lib/agent/runner/harness/pi/shared.ts +++ /dev/null @@ -1,369 +0,0 @@ -/** - * pi session machinery shared by the linear run (index.ts) and the - * orchestrator's per-task runs (task.ts). No typebox in this module graph — - * the SDK always arrives as the caller's lazy import, passed in as `sdk`. - */ - -import { getUI, type SpinnerHandle } from '@ui'; -import { logToFile } from '@utils/debug'; -import { analytics } from '@utils/analytics'; -import { WIZARD_USER_AGENT } from '@lib/constants'; -import { AgentErrorType } from '@lib/agent/agent-interface'; -import { AgentSignals } from '@lib/agent/signals'; -import type { AgentOutputSignals } from '@lib/agent/output-signals'; -import type { BootstrapResult } from '@lib/agent/runner/shared/types'; -import type { ThinkingLevel } from '../../switchboard/models'; -import { buildGatewayProvider, GATEWAY_PROVIDER } from './gateway'; - -/** The lazily imported pi SDK module, passed in by the caller. */ -export type PiSdk = typeof import('@earendil-works/pi-coding-agent'); -export type PiAgentSession = Awaited< - ReturnType ->['session']; -type PiModelRegistry = ReturnType; -type PiModel = NonNullable>; - -/** - * The ONLY environment variables pi's tool subprocesses (bash → npm/pip/…) are - * allowed to see. Everything else — every secret (POSTHOG_PERSONAL_API_KEY, - * ANTHROPIC_*, AWS_*), every ambient credential, the parent process's whole env - * — is dropped before a child is spawned. pi's own gateway auth is programmatic - * (the access token never lives in env), so a minimal env costs the agent - * nothing while closing the leak that exposed the key before. Kept to what a - * package manager genuinely needs to run. - */ -const ALLOWED_SUBPROCESS_ENV_KEYS = [ - 'PATH', - 'HOME', - 'SHELL', - 'USER', - 'LOGNAME', - 'TMPDIR', - 'TMP', - 'TEMP', - 'TERM', - 'LANG', - 'LC_ALL', - 'LC_CTYPE', - 'NODE_EXTRA_CA_CERTS', - 'SSL_CERT_FILE', - 'SSL_CERT_DIR', - 'HTTP_PROXY', - 'HTTPS_PROXY', - 'NO_PROXY', - 'http_proxy', - 'https_proxy', - 'no_proxy', -]; - -/** A fresh subprocess env holding only the allowlisted keys present in process.env. */ -export function buildScrubbedEnv(): NodeJS.ProcessEnv { - const env: NodeJS.ProcessEnv = {}; - for (const key of ALLOWED_SUBPROCESS_ENV_KEYS) { - const value = process.env[key]; - if (value !== undefined) env[key] = value; - } - return env; -} - -/** - * Tag a tool with an execution mode (mutates + returns it). Read-only tools are - * `parallel` so a single turn that batches independent reads/searches runs them - * at once; mutating/install tools are `sequential` so a batch never races writes - * or concurrent installs. pi-agent-core runs a batch in parallel only when no - * tool in it is `sequential`. - */ -export function withMode(tool: T, mode: 'sequential' | 'parallel'): T { - (tool as { executionMode?: 'sequential' | 'parallel' }).executionMode = mode; - return tool; -} - -/** Pull plain text out of a pi AgentMessage (content is text/image blocks). */ -export function extractText(message: unknown): string { - const content = (message as { content?: unknown })?.content; - if (typeof content === 'string') return content; - if (Array.isArray(content)) { - return content - .filter((c): c is { type: string; text: string } => { - const block = c as { type?: string; text?: unknown }; - return block?.type === 'text' && typeof block.text === 'string'; - }) - .map((c) => c.text) - .join(''); - } - return ''; -} - -/** - * Surface `[DASHBOARD_URL]` / `[NOTEBOOK_URL]` markers the agent prints (after - * the MCP creates them) into the outro link, mirroring the anthropic path's - * signal parsing (#9). The marker carries the URL the MCP returned. - */ -export function applyOutroMarkers(textBlock: string): void { - const markers: Array<[string, (url: string) => void]> = [ - [AgentSignals.DASHBOARD_URL, (url) => getUI().setDashboardUrl(url)], - [AgentSignals.NOTEBOOK_URL, (url) => getUI().setNotebookUrl(url)], - ]; - for (const [marker, apply] of markers) { - const idx = textBlock.indexOf(marker); - if (idx === -1) continue; - const url = textBlock - .slice(idx + marker.length) - .trim() - .split(/\s/)[0]; - if (url) apply(url); - } -} - -/** - * The text of the last `[STATUS] …` line in a block, if any. Last wins so the - * spinner shows the most recent action when a turn prints several. - */ -export function lastStatusLine(textBlock: string): string | undefined { - let status: string | undefined; - for (const line of textBlock.split('\n')) { - const idx = line.indexOf(AgentSignals.STATUS); - if (idx !== -1) { - status = line.slice(idx + AgentSignals.STATUS.length).trim(); - } - } - return status || undefined; -} - -/** Register the PostHog gateway on a fresh in-memory registry and resolve the model; undefined when it can't be. */ -export function resolveGatewayModel( - sdk: PiSdk, - boot: BootstrapResult, - modelId: string, - opts: { applyEffortFlag?: boolean; effort?: ThinkingLevel } = {}, -): - | { - registry: PiModelRegistry; - model: PiModel; - caps: ReturnType['caps']; - gatewayUrl: string; - } - | undefined { - const { provider, caps, gatewayUrl } = buildGatewayProvider({ - gatewayUrl: boot.credentials.host.gatewayUrl, - accessToken: boot.credentials.accessToken, - wizardMetadata: boot.wizardMetadata, - wizardFlags: boot.wizardFlags, - modelId, - applyEffortFlag: opts.applyEffortFlag, - effort: opts.effort, - }); - const registry = sdk.ModelRegistry.inMemory(sdk.AuthStorage.create()); - registry.registerProvider(GATEWAY_PROVIDER, provider as never); - const model = registry.find(GATEWAY_PROVIDER, modelId); - if (!model) return undefined; - return { registry, model, caps, gatewayUrl }; -} - -/** Wire the real PostHog MCP, best-effort — on failure the run continues without the posthog_* tools. */ -export async function connectPostHogMcp( - sdk: PiSdk, - boot: BootstrapResult, - tag: string, -): Promise<{ - extensionFactory?: (pi: unknown) => void; - cleanup?: () => void; - instructions?: string; -}> { - try { - const { setupPostHogMcp } = await import('./mcp'); - return await setupPostHogMcp({ - agentDir: sdk.getAgentDir(), - mcpUrl: boot.credentials.host.mcpUrl, - accessToken: boot.credentials.accessToken, - userAgent: WIZARD_USER_AGENT, - }); - } catch (err) { - logToFile(`${tag} PostHog MCP setup skipped: ${String(err)}`); - return {}; - } -} - -/** The stock coding tools rooted at `dir` (modes tagged, bash env-scrubbed), as factories so callers register only what their allow list grants. */ -export function piCodingToolFactories(sdk: PiSdk, dir: string) { - return { - read: () => withMode(sdk.createReadToolDefinition(dir), 'parallel'), - edit: () => withMode(sdk.createEditToolDefinition(dir), 'sequential'), - write: () => withMode(sdk.createWriteToolDefinition(dir), 'sequential'), - bash: () => - withMode( - sdk.createBashToolDefinition(dir, { - spawnHook: (ctx) => ({ ...ctx, env: buildScrubbedEnv() }), - }), - 'sequential', - ), - ls: () => withMode(sdk.createLsToolDefinition(dir), 'parallel'), - find: () => withMode(sdk.createFindToolDefinition(dir), 'parallel'), - grep: () => withMode(sdk.createGrepToolDefinition(dir), 'parallel'), - } as const; -} - -/** A hermetic pi session: nothing loads from the target project, `customTools` is the entire tool surface, extensions are bound before returning. */ -export async function createHermeticSession( - sdk: PiSdk, - opts: { - cwd: string; - systemPrompt: string; - extensionFactories: Array<(pi: unknown) => void>; - model: PiModel; - registry: PiModelRegistry; - thinkingLevel?: ThinkingLevel; - customTools: Parameters[0]['customTools']; - }, -): Promise { - const resourceLoader = new sdk.DefaultResourceLoader({ - cwd: opts.cwd, - agentDir: sdk.getAgentDir(), - systemPrompt: opts.systemPrompt, - noExtensions: true, - noSkills: true, - noContextFiles: true, - noPromptTemplates: true, - noThemes: true, - extensionFactories: opts.extensionFactories, - }); - await resourceLoader.reload(); - - const { session } = await sdk.createAgentSession({ - model: opts.model, - modelRegistry: opts.registry, - thinkingLevel: opts.thinkingLevel, - cwd: opts.cwd, - sessionManager: sdk.SessionManager.inMemory(opts.cwd), - resourceLoader, - noTools: 'builtin', - customTools: opts.customTools, - }); - await session.bindExtensions({}); - return session; -} - -/** Counters the session watcher keeps live; read them after the run. */ -export interface SessionCounts { - assistantTurns: number; - toolCalls: number; -} - -/** Map pi session events onto the spinner, status history, signals, and log, keeping the counters live. */ -export function watchSession( - agentSession: PiAgentSession, - opts: { tag: string; spinner: SpinnerHandle; signals: AgentOutputSignals }, -): { counts: SessionCounts; unsubscribe: () => void } { - const { tag, spinner, signals } = opts; - const counts: SessionCounts = { assistantTurns: 0, toolCalls: 0 }; - const unsubscribe = agentSession.subscribe((event) => { - switch (event.type) { - case 'message_end': { - // User prompts also emit message_end; only assistant turns count. - if ((event.message as { role?: string })?.role !== 'assistant') { - break; - } - counts.assistantTurns += 1; - const assistant = extractText(event.message).trim(); - if (assistant) { - logToFile(`${tag} assistant: ${assistant.slice(0, 1000)}`); - applyOutroMarkers(assistant); - const statusText = lastStatusLine(assistant); - if (statusText) { - getUI().pushStatus(statusText); - spinner.message(statusText); - } - for (const line of assistant.split('\n')) signals.push(line); - } - break; - } - case 'tool_execution_start': { - counts.toolCalls += 1; - const args = JSON.stringify(event.args ?? {}).slice(0, 200); - logToFile(`${tag} → ${event.toolName} ${args}`); - // Don't surface raw tool names in the spinner — the anthropic path - // doesn't, and it reads as noise. - break; - } - case 'tool_execution_end': { - if (event.isError) { - logToFile( - `${tag} ✗ ${event.toolName}: ${String(event.result).slice(0, 300)}`, - ); - } - break; - } - case 'agent_end': { - logToFile(`${tag} agent_end (willRetry=${String(event.willRetry)})`); - break; - } - default: - break; - } - }); - return { counts, unsubscribe }; -} - -/** A run clock: call once at start, call the result for the durations shape. */ -export function startRunClock(): () => { - duration_ms: number; - duration_seconds: number; -} { - const startTime = Date.now(); - return () => { - const durationMs = Date.now() - startTime; - return { - duration_ms: durationMs, - duration_seconds: Math.round(durationMs / 1000), - }; - }; -} - -/** Classify a thrown run error the way both entry points report it. */ -export function classifyRunError(err: unknown): { - error: AgentErrorType; - message: string; -} { - const message = err instanceof Error ? err.message : String(err); - const lower = message.toLowerCase(); - const error = - lower.includes('rate limit') || lower.includes('429') - ? AgentErrorType.RATE_LIMIT - : AgentErrorType.API_ERROR; - return { error, message }; -} - -/** The `agent completed` capture plus one parseable usage log line (`task=` only when a task_type rides along). */ -export function captureAgentUsage(opts: { - tag: string; - modelId: string; - counts: SessionCounts; - durations: { duration_ms: number; duration_seconds: number }; - stats: ReturnType; - analyticsProperties?: Record; -}): void { - const { tag, modelId, counts, durations, stats, analyticsProperties } = opts; - analytics.wizardCapture('agent completed', { - ...durations, - model: modelId, - num_turns: counts.assistantTurns, - // API-reported tokens only; no total_cost_usd — the API returns no - // cost, and $ai_generation already prices the run authoritatively. - input_tokens: stats.tokens.input, - output_tokens: stats.tokens.output, - cache_creation_input_tokens: stats.tokens.cacheWrite, - cache_read_input_tokens: stats.tokens.cacheRead, - ...analyticsProperties, - }); - const taskType = - typeof analyticsProperties?.task_type === 'string' - ? analyticsProperties.task_type - : undefined; - logToFile( - `${tag} usage${taskType ? ` task=${taskType}` : ''} model=${modelId} dur=${ - durations.duration_seconds - }s turns=${counts.assistantTurns} in=${stats.tokens.input} out=${ - stats.tokens.output - } cacheR=${stats.tokens.cacheRead} cacheW=${stats.tokens.cacheWrite}`, - ); -} diff --git a/src/lib/agent/runner/harness/pi/subagent.ts b/src/lib/agent/runner/harness/pi/subagent.ts index 477c884ec..1f5e7f7d6 100644 --- a/src/lib/agent/runner/harness/pi/subagent.ts +++ b/src/lib/agent/runner/harness/pi/subagent.ts @@ -17,7 +17,6 @@ import { Type } from 'typebox'; import { defineTool } from '@earendil-works/pi-coding-agent'; import type { ToolDefinition } from '@earendil-works/pi-coding-agent'; import { logToFile } from '@utils/debug'; -import { extractText } from './shared'; /** * Read-only built-ins a subagent may use. bash is supplied separately as the @@ -40,6 +39,21 @@ function text(s: string): { return { content: [{ type: 'text', text: s }], details: {} }; } +function extractText(message: unknown): string { + const content = (message as { content?: unknown })?.content; + if (typeof content === 'string') return content; + if (Array.isArray(content)) { + return content + .filter((c): c is { type: string; text: string } => { + const b = c as { type?: string; text?: unknown }; + return b?.type === 'text' && typeof b.text === 'string'; + }) + .map((c) => c.text) + .join(''); + } + return ''; +} + export interface SubagentContext { /** Resolved gateway model (same as the parent). */ // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/src/lib/agent/runner/harness/pi/task.ts b/src/lib/agent/runner/harness/pi/task.ts index a2bd1ad0d..de9f72685 100644 --- a/src/lib/agent/runner/harness/pi/task.ts +++ b/src/lib/agent/runner/harness/pi/task.ts @@ -1,8 +1,10 @@ /** * Orchestrator-mode execution on pi: one fresh pi session per unit of work — - * the seed plan, or one drained task. The session machinery lives in - * `shared.ts`; this module configures the leaner per-task session: the task's - * allowed coding tools, the wizard env tools, and the queue tools. + * the seed plan, or one drained task. The linear pipeline's concerns (skill + * menu, todo panel, event-plan cleanup) stay in `index.ts`; this module builds + * the leaner per-task session: gateway model, security fence, the task's + * allowed coding tools, the wizard env tools, and the in-process orchestrator + * queue tools. * * The task's `allowedTools` / `disallowedTools` arrive in the wizard's tool * vocabulary (`Read`, `Edit`, `Glob`, …, plus MCP-qualified orchestrator names @@ -14,25 +16,24 @@ * Loaded lazily from `index.ts` (typebox/ESM constraint, same as tools.ts). */ +import { getUI } from '@ui'; import { logToFile } from '@utils/debug'; import { analytics } from '@utils/analytics'; -import { WIZARD_REMARK_EVENT_NAME } from '@lib/constants'; +import { WIZARD_REMARK_EVENT_NAME, WIZARD_USER_AGENT } from '@lib/constants'; import { AgentErrorType } from '@lib/agent/agent-interface'; import { REMARK_INSTRUCTION } from '@lib/agent/signals'; import { AgentOutputSignals } from '@lib/agent/output-signals'; import { TaskStatus } from '../../sequence/orchestrator/queue'; import type { OrchestratorToolsContext } from '../../sequence/orchestrator/queue-tools'; import type { AgentResult, TaskRunInputs } from '../types'; +import { buildGatewayProvider, GATEWAY_PROVIDER } from './gateway'; import { - captureAgentUsage, - classifyRunError, - connectPostHogMcp, - createHermeticSession, - piCodingToolFactories, - resolveGatewayModel, - startRunClock, - watchSession, -} from './shared'; + applyOutroMarkers, + buildScrubbedEnv, + extractText, + lastStatusLine, + withMode, +} from './index'; /** wizard tool vocabulary → the pi tool definitions it unlocks. */ const CODING_TOOL_MAP: Record = { @@ -166,8 +167,16 @@ export async function runPiTask(inputs: TaskRunInputs): Promise { if (spinnerMessage) spinner.start(spinnerMessage); + const startTime = Date.now(); const signals = new AgentOutputSignals(); - const runDurations = startRunClock(); + let assistantTurns = 0; + const runDurations = () => { + const durationMs = Date.now() - startTime; + return { + duration_ms: durationMs, + duration_seconds: Math.round(durationMs / 1000), + }; + }; const captureAborted = () => analytics.wizardCapture('agent aborted', { ...runDurations(), @@ -177,20 +186,42 @@ export async function runPiTask(inputs: TaskRunInputs): Promise { try { const sdk = await import('@earendil-works/pi-coding-agent'); + const { + createAgentSession, + DefaultResourceLoader, + SessionManager, + AuthStorage, + ModelRegistry, + getAgentDir, + createLsToolDefinition, + createFindToolDefinition, + createGrepToolDefinition, + createBashToolDefinition, + createReadToolDefinition, + createEditToolDefinition, + createWriteToolDefinition, + } = sdk; - // Per-task agents own their effort via the prompt frontmatter (falling - // back to the model table), not the run-wide wizard-pi-effort flag. - const gateway = resolveGatewayModel(sdk, boot, modelId, { + const { provider, caps, gatewayUrl } = buildGatewayProvider({ + gatewayUrl: boot.credentials.host.gatewayUrl, + accessToken: boot.credentials.accessToken, + wizardMetadata: boot.wizardMetadata, + wizardFlags: boot.wizardFlags, + modelId, + // Per-task agents own their effort via the prompt frontmatter (falling back + // to the model table), not the run-wide wizard-pi-effort flag. applyEffortFlag: false, effort, }); - if (!gateway) { + const registry = ModelRegistry.inMemory(AuthStorage.create()); + registry.registerProvider(GATEWAY_PROVIDER, provider as never); + const model = registry.find(GATEWAY_PROVIDER, modelId); + if (!model) { return { error: AgentErrorType.API_ERROR, message: 'pi: gateway model could not be resolved', }; } - const { registry, model, caps, gatewayUrl } = gateway; // The same fail-closed fence as the linear run, with the task's disallow // list layered in (both the wizard-vocabulary and pi-short names). @@ -210,19 +241,63 @@ export async function runPiTask(inputs: TaskRunInputs): Promise { const extensionFactories = [security.factory] as Array< (pi: unknown) => void >; - const mcp = await connectPostHogMcp(sdk, boot, '[pi-task]'); - if (mcp.extensionFactory) extensionFactories.push(mcp.extensionFactory); - const mcpCleanup = mcp.cleanup; - const posthogMcp = mcp.extensionFactory !== undefined; + let mcpCleanup: (() => void) | undefined; + let posthogMcp = false; + try { + const { setupPostHogMcp } = await import('./mcp'); + const mcp = await setupPostHogMcp({ + agentDir: getAgentDir(), + mcpUrl: boot.credentials.host.mcpUrl, + accessToken: boot.credentials.accessToken, + userAgent: WIZARD_USER_AGENT, + }); + extensionFactories.push(mcp.extensionFactory); + mcpCleanup = mcp.cleanup; + posthogMcp = true; + } catch (err) { + logToFile(`[pi-task] PostHog MCP setup skipped: ${String(err)}`); + } const codingTools = allowedPiCodingTools(allowedTools); const orchestratorTools = allowedOrchestratorTools(disallowedTools); + const { getWizardCommandments } = await import('@lib/agent/commandments'); + const resourceLoader = new DefaultResourceLoader({ + cwd: session.installDir, + agentDir: getAgentDir(), + systemPrompt: + getWizardCommandments() + + '\n' + + taskRuntimeNotes({ bash: codingTools.has('bash'), posthogMcp }), + noExtensions: true, + noSkills: true, + noContextFiles: true, + noPromptTemplates: true, + noThemes: true, + extensionFactories, + }); + await resourceLoader.reload(); + // The task's coding tools, gated by its allow list. Reads and searches run // in parallel; mutating tools stay sequential. Bash subprocesses get the // scrubbed env, same as the linear run. const dir = session.installDir; - const codingToolDefs = Object.entries(piCodingToolFactories(sdk, dir)) + const codingToolFactories = { + read: () => withMode(createReadToolDefinition(dir), 'parallel'), + edit: () => withMode(createEditToolDefinition(dir), 'sequential'), + write: () => withMode(createWriteToolDefinition(dir), 'sequential'), + bash: () => + withMode( + createBashToolDefinition(dir, { + spawnHook: (ctx) => ({ ...ctx, env: buildScrubbedEnv() }), + }), + 'sequential', + ), + ls: () => withMode(createLsToolDefinition(dir), 'parallel'), + find: () => withMode(createFindToolDefinition(dir), 'parallel'), + grep: () => withMode(createGrepToolDefinition(dir), 'parallel'), + } as const; + const codingToolDefs = Object.entries(codingToolFactories) .filter(([name]) => codingTools.has(name)) .map(([, make]) => make()); @@ -243,24 +318,58 @@ export async function runPiTask(inputs: TaskRunInputs): Promise { orchestratorTools.has(t.name), ); - const { getWizardCommandments } = await import('@lib/agent/commandments'); - const agentSession = await createHermeticSession(sdk, { - cwd: dir, - systemPrompt: - getWizardCommandments() + - '\n' + - taskRuntimeNotes({ bash: codingTools.has('bash'), posthogMcp }), - extensionFactories, + const { session: agentSession } = await createAgentSession({ model, - registry, + modelRegistry: registry, thinkingLevel: caps.thinkingLevel, + cwd: dir, + sessionManager: SessionManager.inMemory(dir), + resourceLoader, + noTools: 'builtin', customTools: [...codingToolDefs, ...wizardTools, ...queueTools], }); + await agentSession.bindExtensions({}); - const { counts, unsubscribe } = watchSession(agentSession, { - tag: '[pi-task]', - spinner, - signals, + const unsubscribe = agentSession.subscribe((event) => { + switch (event.type) { + case 'message_end': { + // User prompts also emit message_end; only assistant turns count. + if ((event.message as { role?: string })?.role !== 'assistant') { + break; + } + assistantTurns += 1; + const assistant = extractText(event.message).trim(); + if (assistant) { + logToFile(`[pi-task] assistant: ${assistant.slice(0, 1000)}`); + applyOutroMarkers(assistant); + const statusText = lastStatusLine(assistant); + if (statusText) { + getUI().pushStatus(statusText); + spinner.message(statusText); + } + for (const line of assistant.split('\n')) signals.push(line); + } + break; + } + case 'tool_execution_start': { + const args = JSON.stringify(event.args ?? {}).slice(0, 200); + logToFile(`[pi-task] → ${event.toolName} ${args}`); + break; + } + case 'tool_execution_end': { + if (event.isError) { + logToFile( + `[pi-task] ✗ ${event.toolName}: ${String(event.result).slice( + 0, + 300, + )}`, + ); + } + break; + } + default: + break; + } }); try { @@ -310,23 +419,41 @@ export async function runPiTask(inputs: TaskRunInputs): Promise { analytics.capture(WIZARD_REMARK_EVENT_NAME, { remark }); } - captureAgentUsage({ - tag: '[pi-task]', - modelId, - counts, - durations: runDurations(), - stats: agentSession.getSessionStats(), - analyticsProperties, + const stats = agentSession.getSessionStats(); + const durations = runDurations(); + analytics.wizardCapture('agent completed', { + ...durations, + model: modelId, + num_turns: assistantTurns, + input_tokens: stats.tokens.input, + output_tokens: stats.tokens.output, + cache_creation_input_tokens: stats.tokens.cacheWrite, + cache_read_input_tokens: stats.tokens.cacheRead, + ...analyticsProperties, }); + // Per-task usage on one parseable line so a run's per-task time and cost are + // observable from the log, not only from analytics. + const taskType = + typeof (analyticsProperties as { task_type?: unknown })?.task_type === + 'string' + ? (analyticsProperties as { task_type: string }).task_type + : modelId; + logToFile( + `[pi-task] usage task=${taskType} model=${modelId} dur=${durations.duration_seconds}s turns=${assistantTurns} in=${stats.tokens.input} out=${stats.tokens.output} cacheR=${stats.tokens.cacheRead} cacheW=${stats.tokens.cacheWrite}`, + ); if (successMessage) spinner.stop(successMessage); return {}; } catch (err) { - const { error, message } = classifyRunError(err); + const message = err instanceof Error ? err.message : String(err); logToFile(`[pi-task] run error: ${message}`); if (errorMessage || spinnerMessage) { spinner.stop(errorMessage ?? 'Task failed'); } captureAborted(); - return { error, message }; + const lower = message.toLowerCase(); + if (lower.includes('rate limit') || lower.includes('429')) { + return { error: AgentErrorType.RATE_LIMIT, message }; + } + return { error: AgentErrorType.API_ERROR, message }; } } From ea7518284118510e2faecdbc31a2c6aa2f4072e0 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Wed, 15 Jul 2026 12:43:41 -0400 Subject: [PATCH 5/8] refactor: apply /simplify findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - taskModelSpec takes the full HarnessPick and absorbs the switchboard fallback, so the enqueue-override -> frontmatter -> pick precedence lives in one place instead of being restated at each call site. - onTransition builds its analytics base lazily — enqueue/requeue never read it, and resolveHarness logs per call. - resolveReferenceSkillId (one-line wrapper, single call site) inlined. Generated-By: PostHog Code Task-Id: fafc230d-6f14-4e4d-9462-0e7f18a1eec1 --- .../__tests__/agent-prompt-loader.test.ts | 45 +++++++++++-------- src/lib/agent/agent-prompt-loader.ts | 15 ++++--- .../orchestrator/orchestrator-runner.ts | 39 +++++++--------- 3 files changed, 51 insertions(+), 48 deletions(-) diff --git a/src/lib/agent/__tests__/agent-prompt-loader.test.ts b/src/lib/agent/__tests__/agent-prompt-loader.test.ts index 5ba580ffd..ece82d1a3 100644 --- a/src/lib/agent/__tests__/agent-prompt-loader.test.ts +++ b/src/lib/agent/__tests__/agent-prompt-loader.test.ts @@ -234,19 +234,27 @@ describe('resolveTask', () => { it('resolves per-harness model + effort from the prompt', () => { const registry = registryOf([prompt]); const task = store.enqueue({ type: 'capture' }); - expect(taskModelSpec(registry, task, Harness.pi)).toEqual({ + expect( + taskModelSpec(registry, task, { harness: Harness.pi, model: 'pick-m' }), + ).toEqual({ model: 'openai/gpt-5.6-luna', effort: 'low', }); - expect(taskModelSpec(registry, task, Harness.anthropic).model).toBe( - 'claude-haiku-4-5-20251001', - ); + expect( + taskModelSpec(registry, task, { + harness: Harness.anthropic, + model: 'pick-m', + }).model, + ).toBe('claude-haiku-4-5-20251001'); }); it('prefers the enqueue model override over the prompt model', () => { const registry = registryOf([prompt]); const task = store.enqueue({ type: 'capture', model: 'override-x' }); - expect(taskModelSpec(registry, task, Harness.pi).model).toBe('override-x'); + expect( + taskModelSpec(registry, task, { harness: Harness.pi, model: 'pick-m' }) + .model, + ).toBe('override-x'); }); it("appends upstream dependencies' handoffs as context", () => { @@ -322,26 +330,27 @@ describe('taskModelSpec', () => { 'capture', ); - it('prefers the enqueue override, then the prompt; no default baked in', () => { + it('prefers the enqueue override, then the prompt, then the switchboard pick', () => { const registry = registryOf([prompt]); const task = { type: 'capture' }; + const pick = { harness: Harness.pi, model: 'pick-m' }; expect( - taskModelSpec( - registry, - { ...task, model: 'override' } as never, - Harness.pi, - ).model, + taskModelSpec(registry, { ...task, model: 'override' } as never, pick) + .model, ).toBe('override'); - expect(taskModelSpec(registry, task as never, Harness.pi).model).toBe( + expect(taskModelSpec(registry, task as never, pick).model).toBe( 'prompt-model', ); - // An empty column stays undefined — the caller falls back to its switchboard pick. - expect( - taskModelSpec(registry, task as never, Harness.anthropic).model, - ).toBeUndefined(); + // An empty column falls back to the pick, per harness. expect( - taskModelSpec(registryOf([]), task as never, Harness.pi).model, - ).toBeUndefined(); + taskModelSpec(registry, task as never, { + harness: Harness.anthropic, + model: 'pick-m', + }).model, + ).toBe('pick-m'); + expect(taskModelSpec(registryOf([]), task as never, pick).model).toBe( + 'pick-m', + ); }); }); diff --git a/src/lib/agent/agent-prompt-loader.ts b/src/lib/agent/agent-prompt-loader.ts index 1658dc418..8cfdc036c 100644 --- a/src/lib/agent/agent-prompt-loader.ts +++ b/src/lib/agent/agent-prompt-loader.ts @@ -21,6 +21,7 @@ import type { } from './runner/sequence/orchestrator/queue'; import type { ResolvedTask } from './runner/sequence/orchestrator/executor'; import type { HostResolution } from '@lib/host-resolution'; +import type { HarnessPick } from './runner/switchboard'; import { Harness } from '@lib/constants'; import { isThinkingLevel, @@ -397,18 +398,18 @@ export function resolveTask( }; } -/** Enqueue override, then per-profile frontmatter; no default — the caller falls back to its switchboard pick. */ +/** Enqueue override, then per-profile frontmatter, then the switchboard pick — the whole precedence in one place. */ export function taskModelSpec( registry: AgentRegistry, task: QueuedTask, - harness: Harness, -): { model?: string; effort?: ThinkingLevel } { + pick: HarnessPick, +): { model: string; effort?: ThinkingLevel } { const prompt = registry.get(task.type); - const picked = prompt - ? promptModelFor(prompt, harness) - : { model: undefined, effort: undefined }; + const picked: { model?: string; effort?: ThinkingLevel } = prompt + ? promptModelFor(prompt, pick.harness) + : {}; return { - model: task.model ?? picked.model, + model: task.model ?? picked.model ?? pick.model, effort: picked.effort, }; } diff --git a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts index 6319e1638..6e0003c5d 100644 --- a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -106,18 +106,6 @@ export function resolveSkillVariantId( return (family.find((e) => e.default) ?? family[0])?.id; } -/** - * The framework reference is the full `integration` skill. `session.skillId` is - * the bare framework (e.g. `django`), but the skill menu ids it as - * `integration-`. - */ -function resolveReferenceSkillId( - entries: readonly SkillEntry[], - framework: string, -): string | undefined { - return resolveSkillVariantId(entries, 'integration', framework); -} - export async function runOrchestrator( session: WizardSession, programConfig: ProgramConfig, @@ -162,12 +150,16 @@ export async function runOrchestrator( const store = new QueueStore(session.installDir, runId, { onTransition: (event, task) => { - const pick = resolveHarness(switchboardCtx, task.type); - const base = { + // Lazy — enqueue/requeue never read it, and resolveHarness logs per call. + const base = () => ({ type: task.type, - model: taskModelSpec(registry, task, pick.harness).model ?? pick.model, + model: taskModelSpec( + registry, + task, + resolveHarness(switchboardCtx, task.type), + ).model, attempts: task.attempts, - }; + }); switch (event) { case 'enqueue': analytics.wizardCapture('orchestrator task enqueued', { @@ -178,28 +170,28 @@ export async function runOrchestrator( break; case 'start': analytics.wizardCapture('orchestrator task started', { - ...base, + ...base(), ...metrics.recordStart(Date.now()), }); break; case 'complete': metrics.recordComplete(Date.now()); analytics.wizardCapture('orchestrator task completed', { - ...base, + ...base(), duration_ms: durationMs(task), }); break; case 'skip': metrics.recordTerminal(Date.now()); analytics.wizardCapture('orchestrator task skipped', { - ...base, + ...base(), duration_ms: durationMs(task), }); break; case 'fail': metrics.recordTerminal(Date.now()); analytics.wizardCapture('orchestrator task failed', { - ...base, + ...base(), duration_ms: durationMs(task), error: task.error?.type, }); @@ -217,8 +209,9 @@ export async function runOrchestrator( let examplePath: string | undefined; let commandmentsPath: string | undefined; const menuSkillEntries = await fetchSkillMenuEntries(boot.skillsBaseUrl); + // `session.skillId` is the bare framework (e.g. `django`); the menu ids the reference as `integration-`. const referenceSkillId = session.skillId - ? resolveReferenceSkillId(menuSkillEntries, session.skillId) + ? resolveSkillVariantId(menuSkillEntries, 'integration', session.skillId) : undefined; if (referenceSkillId) { const ref = await installSkillById( @@ -414,14 +407,14 @@ export async function runOrchestrator( // per-agent overrides. Prompt-frontmatter model still wins (§3.6). const taskPick = resolveHarness(switchboardCtx, task.type); const taskHarness = requireTaskHarness(taskPick); - const taskModel = taskModelSpec(registry, task, taskPick.harness); + const taskModel = taskModelSpec(registry, task, taskPick); await taskHarness.runTask({ session, programConfig, boot, prompt: assembleTaskPrompt(promptContext, resolved.prompt, skillPaths), spinner, - model: taskModel.model ?? taskPick.model, + model: taskModel.model, effort: taskModel.effort, allowedTools: resolved.allowedTools, disallowedTools: resolved.disallowedTools, From 8f2b40bfe2b4c976cb542b015a07cf354441babb Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Wed, 15 Jul 2026 12:48:47 -0400 Subject: [PATCH 6/8] chore: cut the PR back to the essential fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops the /simplify reshaping (HarnessPick signature, lazy analytics base, wrapper inlining) and the Harness-enum param typing — taskModelSpec keeps its base shape minus the baked-in default, and call sites keep their existing pick fallback. Generated-By: PostHog Code Task-Id: fafc230d-6f14-4e4d-9462-0e7f18a1eec1 --- .../__tests__/agent-prompt-loader.test.ts | 45 +++++++------------ src/lib/agent/agent-prompt-loader.ts | 25 ++++++----- .../orchestrator/orchestrator-runner.ts | 39 +++++++++------- 3 files changed, 52 insertions(+), 57 deletions(-) diff --git a/src/lib/agent/__tests__/agent-prompt-loader.test.ts b/src/lib/agent/__tests__/agent-prompt-loader.test.ts index ece82d1a3..88eb4cde0 100644 --- a/src/lib/agent/__tests__/agent-prompt-loader.test.ts +++ b/src/lib/agent/__tests__/agent-prompt-loader.test.ts @@ -15,7 +15,6 @@ import { } from '../agent-prompt-loader'; import { QueueStore } from '@lib/agent/runner/sequence/orchestrator/queue'; import { HostResolution } from '@lib/host-resolution'; -import { Harness } from '@lib/constants'; function tmpDir(): string { return fs.mkdtempSync(path.join(os.tmpdir(), 'agent-loader-test-')); @@ -58,11 +57,11 @@ Add at least one capture call. it('resolves the per-harness model + effort, not 1:1 across providers', () => { const p = parseAgentPrompt(sample, 'fallback'); - expect(promptModelFor(p, Harness.pi)).toEqual({ + expect(promptModelFor(p, 'pi')).toEqual({ model: 'openai/gpt-5.6-terra', effort: 'medium', }); - expect(promptModelFor(p, Harness.anthropic)).toEqual({ + expect(promptModelFor(p, 'anthropic')).toEqual({ model: 'claude-sonnet-4-6', effort: undefined, }); @@ -234,27 +233,19 @@ describe('resolveTask', () => { it('resolves per-harness model + effort from the prompt', () => { const registry = registryOf([prompt]); const task = store.enqueue({ type: 'capture' }); - expect( - taskModelSpec(registry, task, { harness: Harness.pi, model: 'pick-m' }), - ).toEqual({ + expect(taskModelSpec(registry, task, 'pi')).toEqual({ model: 'openai/gpt-5.6-luna', effort: 'low', }); - expect( - taskModelSpec(registry, task, { - harness: Harness.anthropic, - model: 'pick-m', - }).model, - ).toBe('claude-haiku-4-5-20251001'); + expect(taskModelSpec(registry, task, 'anthropic').model).toBe( + 'claude-haiku-4-5-20251001', + ); }); it('prefers the enqueue model override over the prompt model', () => { const registry = registryOf([prompt]); const task = store.enqueue({ type: 'capture', model: 'override-x' }); - expect( - taskModelSpec(registry, task, { harness: Harness.pi, model: 'pick-m' }) - .model, - ).toBe('override-x'); + expect(taskModelSpec(registry, task, 'pi').model).toBe('override-x'); }); it("appends upstream dependencies' handoffs as context", () => { @@ -330,27 +321,23 @@ describe('taskModelSpec', () => { 'capture', ); - it('prefers the enqueue override, then the prompt, then the switchboard pick', () => { + it('prefers the enqueue override, then the prompt; the switchboard pick is the caller fallback', () => { const registry = registryOf([prompt]); const task = { type: 'capture' }; - const pick = { harness: Harness.pi, model: 'pick-m' }; expect( - taskModelSpec(registry, { ...task, model: 'override' } as never, pick) + taskModelSpec(registry, { ...task, model: 'override' } as never, 'pi') .model, ).toBe('override'); - expect(taskModelSpec(registry, task as never, pick).model).toBe( + expect(taskModelSpec(registry, task as never, 'pi').model).toBe( 'prompt-model', ); - // An empty column falls back to the pick, per harness. + // An empty column stays undefined — the caller falls back to its switchboard pick. expect( - taskModelSpec(registry, task as never, { - harness: Harness.anthropic, - model: 'pick-m', - }).model, - ).toBe('pick-m'); - expect(taskModelSpec(registryOf([]), task as never, pick).model).toBe( - 'pick-m', - ); + taskModelSpec(registry, task as never, 'anthropic').model, + ).toBeUndefined(); + expect( + taskModelSpec(registryOf([]), task as never, 'pi').model, + ).toBeUndefined(); }); }); diff --git a/src/lib/agent/agent-prompt-loader.ts b/src/lib/agent/agent-prompt-loader.ts index 8cfdc036c..3dca17c87 100644 --- a/src/lib/agent/agent-prompt-loader.ts +++ b/src/lib/agent/agent-prompt-loader.ts @@ -21,8 +21,6 @@ import type { } from './runner/sequence/orchestrator/queue'; import type { ResolvedTask } from './runner/sequence/orchestrator/executor'; import type { HostResolution } from '@lib/host-resolution'; -import type { HarnessPick } from './runner/switchboard'; -import { Harness } from '@lib/constants'; import { isThinkingLevel, type ThinkingLevel, @@ -142,9 +140,9 @@ export interface AgentPrompt { * column, anything else the sdk (anthropic) column. */ export function promptModelFor( prompt: AgentPrompt, - harness: Harness, + harness: string, ): { model?: string; effort?: ThinkingLevel } { - const pi = harness === Harness.pi; + const pi = harness === 'pi'; return { model: pi ? prompt.modelPi : prompt.modelSdk, effort: pi ? prompt.effortPi : prompt.effortSdk, @@ -398,18 +396,21 @@ export function resolveTask( }; } -/** Enqueue override, then per-profile frontmatter, then the switchboard pick — the whole precedence in one place. */ +/** The model + effort a task runs on for a harness: enqueue override, then the + * prompt's per-profile frontmatter; the caller's switchboard pick is the fallback. */ export function taskModelSpec( registry: AgentRegistry, task: QueuedTask, - pick: HarnessPick, -): { model: string; effort?: ThinkingLevel } { - const prompt = registry.get(task.type); - const picked: { model?: string; effort?: ThinkingLevel } = prompt - ? promptModelFor(prompt, pick.harness) - : {}; + harness: string, +): { model?: string; effort?: ThinkingLevel } { + const picked = promptModelFor( + registry.get(task.type) ?? EMPTY_PROMPT, + harness, + ); return { - model: task.model ?? picked.model ?? pick.model, + model: task.model ?? picked.model, effort: picked.effort, }; } + +const EMPTY_PROMPT = {} as AgentPrompt; diff --git a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts index 6e0003c5d..6319e1638 100644 --- a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -106,6 +106,18 @@ export function resolveSkillVariantId( return (family.find((e) => e.default) ?? family[0])?.id; } +/** + * The framework reference is the full `integration` skill. `session.skillId` is + * the bare framework (e.g. `django`), but the skill menu ids it as + * `integration-`. + */ +function resolveReferenceSkillId( + entries: readonly SkillEntry[], + framework: string, +): string | undefined { + return resolveSkillVariantId(entries, 'integration', framework); +} + export async function runOrchestrator( session: WizardSession, programConfig: ProgramConfig, @@ -150,16 +162,12 @@ export async function runOrchestrator( const store = new QueueStore(session.installDir, runId, { onTransition: (event, task) => { - // Lazy — enqueue/requeue never read it, and resolveHarness logs per call. - const base = () => ({ + const pick = resolveHarness(switchboardCtx, task.type); + const base = { type: task.type, - model: taskModelSpec( - registry, - task, - resolveHarness(switchboardCtx, task.type), - ).model, + model: taskModelSpec(registry, task, pick.harness).model ?? pick.model, attempts: task.attempts, - }); + }; switch (event) { case 'enqueue': analytics.wizardCapture('orchestrator task enqueued', { @@ -170,28 +178,28 @@ export async function runOrchestrator( break; case 'start': analytics.wizardCapture('orchestrator task started', { - ...base(), + ...base, ...metrics.recordStart(Date.now()), }); break; case 'complete': metrics.recordComplete(Date.now()); analytics.wizardCapture('orchestrator task completed', { - ...base(), + ...base, duration_ms: durationMs(task), }); break; case 'skip': metrics.recordTerminal(Date.now()); analytics.wizardCapture('orchestrator task skipped', { - ...base(), + ...base, duration_ms: durationMs(task), }); break; case 'fail': metrics.recordTerminal(Date.now()); analytics.wizardCapture('orchestrator task failed', { - ...base(), + ...base, duration_ms: durationMs(task), error: task.error?.type, }); @@ -209,9 +217,8 @@ export async function runOrchestrator( let examplePath: string | undefined; let commandmentsPath: string | undefined; const menuSkillEntries = await fetchSkillMenuEntries(boot.skillsBaseUrl); - // `session.skillId` is the bare framework (e.g. `django`); the menu ids the reference as `integration-`. const referenceSkillId = session.skillId - ? resolveSkillVariantId(menuSkillEntries, 'integration', session.skillId) + ? resolveReferenceSkillId(menuSkillEntries, session.skillId) : undefined; if (referenceSkillId) { const ref = await installSkillById( @@ -407,14 +414,14 @@ export async function runOrchestrator( // per-agent overrides. Prompt-frontmatter model still wins (§3.6). const taskPick = resolveHarness(switchboardCtx, task.type); const taskHarness = requireTaskHarness(taskPick); - const taskModel = taskModelSpec(registry, task, taskPick); + const taskModel = taskModelSpec(registry, task, taskPick.harness); await taskHarness.runTask({ session, programConfig, boot, prompt: assembleTaskPrompt(promptContext, resolved.prompt, skillPaths), spinner, - model: taskModel.model, + model: taskModel.model ?? taskPick.model, effort: taskModel.effort, allowedTools: resolved.allowedTools, disallowedTools: resolved.disallowedTools, From c35415e5cad804fb7dfa6add6c087a8bc05ad98f Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Wed, 15 Jul 2026 13:05:10 -0400 Subject: [PATCH 7/8] feat: capture 'agent prompt invalid effort' when frontmatter effort is dropped Generated-By: PostHog Code Task-Id: fafc230d-6f14-4e4d-9462-0e7f18a1eec1 --- src/lib/agent/agent-prompt-loader.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib/agent/agent-prompt-loader.ts b/src/lib/agent/agent-prompt-loader.ts index 3dca17c87..239880ae2 100644 --- a/src/lib/agent/agent-prompt-loader.ts +++ b/src/lib/agent/agent-prompt-loader.ts @@ -26,6 +26,7 @@ import { type ThinkingLevel, } from './runner/switchboard/models'; import { logToFile } from '@utils/debug'; +import { analytics } from '@utils/analytics'; /** * The basics the client injects around every agent-prompt body. The `/agents/` @@ -248,6 +249,11 @@ export function parseAgentPrompt( logToFile( `[agent-prompt] ${fallbackType}: ignoring invalid ${key} "${String(v)}"`, ); + analytics.wizardCapture('agent prompt invalid effort', { + task_type: fallbackType, + key, + value: String(v), + }); return undefined; }; return { From 1c9c6a62c19a7be14e02f097dbdf87475dc78ea6 Mon Sep 17 00:00:00 2001 From: "Vincent (Wen Yu) Ge" Date: Wed, 15 Jul 2026 13:37:14 -0400 Subject: [PATCH 8/8] feat(orchestrator): crash on missing skill variant in dev/CI builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preflight already logs + captures a missing skill variant on every build. Now, in dev and CI builds, it also throws so the gap fails a test run loudly instead of passing as a silent skill-less (zero-diff) integration. The throw is gated behind !IS_PRODUCTION_BUILD, which tsdown inlines to a literal, so it is tree-shaken out of the published bundle — verified: the throw string is absent from `pnpm build` output and present in `pnpm build:ci` output, while the log + analytics capture survive in both. Generated-By: PostHog Code Task-Id: fafc230d-6f14-4e4d-9462-0e7f18a1eec1 --- .../orchestrator/orchestrator-runner.ts | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts index 6319e1638..61b737113 100644 --- a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -13,6 +13,7 @@ import { randomUUID } from 'crypto'; import { existsSync, rmSync } from 'fs'; import * as path from 'path'; +import { IS_PRODUCTION_BUILD } from '@env'; import { OutroKind, type WizardSession } from '@lib/wizard-session'; import { installSkillById, @@ -247,23 +248,40 @@ export async function runOrchestrator( ); } - // Preflight every task's mini-skills so a missing variant surfaces before any agent runs. + // Preflight every task's mini-skills. A missing variant means the task runs + // skill-less — a silent zero-diff — so log + capture it on every build. In + // dev and CI the run then crashes so the gap can't slip through a test pass; + // the throw sits behind !IS_PRODUCTION_BUILD, which tsdown inlines to a + // literal, so it is stripped from the published bundle (real users get the + // degraded run, never a crash). + const missingVariants: string[] = []; for (const type of registry.types) { for (const skillId of registry.get(type)?.skills ?? []) { - if (!resolveSkillVariantId(menuSkillEntries, skillId, session.skillId)) { - logToFile( - `[orchestrator] no skill variant type=${type} skill=${skillId} framework=${ - session.skillId ?? 'none' - }`, - ); - analytics.wizardCapture('orchestrator skill variant missing', { - task_type: type, - skill: skillId, - framework: session.skillId, - }); + if (resolveSkillVariantId(menuSkillEntries, skillId, session.skillId)) { + continue; } + missingVariants.push(`${type}/${skillId}`); + logToFile( + `[orchestrator] no skill variant type=${type} skill=${skillId} framework=${ + session.skillId ?? 'none' + }`, + ); + analytics.wizardCapture('orchestrator skill variant missing', { + task_type: type, + skill: skillId, + framework: session.skillId, + }); } } + if (!IS_PRODUCTION_BUILD && missingVariants.length > 0) { + throw new Error( + `Orchestrator preflight: no skill variant for ${missingVariants.join( + ', ', + )} (framework=${ + session.skillId ?? 'none' + }) — fix the context-mill menu or the framework mapping.`, + ); + } // The client injects the basics (project context + the I/O contract) around // every authored agent-prompt body.