diff --git a/APEX_PLAN.md b/APEX_PLAN.md index afdd216..65966e9 100644 --- a/APEX_PLAN.md +++ b/APEX_PLAN.md @@ -87,6 +87,10 @@ this order: first run always works. 6. **Spec forge.** A developer tool that drafts a curated spec from provider docs, checks it against live calls, and leaves a person to approve it. + Status: drafts with Claude, OpenAI or Gemini (`--provider`). Measured once + on 2026-10-12 with Gemini 2.5 Pro: Open Library search docs, passed the + checks in round 1, both operations answered live. Claude and OpenAI are + tested with stubs only. 7. **Data plant.** Server-side house keys, used only where a provider's terms allow an application key, behind a shared cache with per-user quotas and provenance on every value. Lifts the "broad provider catalog" cut in diff --git a/scripts/forge-spec.ts b/scripts/forge-spec.ts index b1eac87..425ab52 100644 --- a/scripts/forge-spec.ts +++ b/scripts/forge-spec.ts @@ -2,6 +2,7 @@ // FLYSPACE: SPEC FORGE // // npm run forge:spec -- --slug openlibrary --docs https://openlibrary.org/dev/docs/api/search [--docs ...] [--key-env NAME] +// [--provider anthropic|openai|google] [--model NAME] // // APEX_PLAN 6.6. Drafts a curated OpenAPI spec from a provider's own docs, // checks it mechanically (specDraft.ts: compiles, reads only, every required @@ -15,17 +16,20 @@ // It never registers anything. A person reads the draft against the docs, // then adds it to curatedApis.ts by hand; that review is the approval. // -// Needs Anthropic credentials (ANTHROPIC_API_KEY, or an `ant auth login` -// profile). The fetched docs are sent to Claude as data; nothing else is. +// The draft comes from Claude (ANTHROPIC_API_KEY, or an `ant auth login` +// profile), OpenAI (OPENAI_API_KEY) or Gemini (GOOGLE_API_KEY): --provider +// picks one, otherwise the first with a key, in that order (scripts/lib/drafters.ts). +// The fetched docs are sent to that provider as data; nothing else is. +// Whatever drafts it, the same checks and live calls decide whether it passes. // ============================================ import { mkdirSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import Anthropic from '@anthropic-ai/sdk'; import { checkDraft, docsText, jsonObjectIn } from '../src/core/capabilities/specDraft'; +import { chooseProvider, draftSession, DraftRefused } from './lib/drafters'; import { verifySpec } from './lib/verifySpec'; -const MODEL = 'claude-opus-5-5'; const MAX_DOC_CHARS = 150_000; const MAX_DOCS = 5; const REPAIR_ROUNDS = 2; @@ -66,28 +70,6 @@ async function fetchDoc(url: string): Promise<{ url: string; text: string }> { return { url, text: text.slice(0, MAX_DOC_CHARS) }; } -async function ask(client: Anthropic, messages: Anthropic.Beta.BetaMessageParam[]): Promise { - const stream = client.beta.messages.stream({ - model: MODEL, - max_tokens: 64_000, - system: SYSTEM, - thinking: { type: 'adaptive' }, - output_config: { effort: 'high' }, - // On a policy decline, re-run on a suitable model instead of stopping. - betas: ['server-side-fallback-2026-07-01'], - fallbacks: 'default', - messages - } as Parameters[0]); - const message = await stream.finalMessage(); - if (message.stop_reason === 'refusal') throw new Error('Claude declined to draft this spec'); - if (message.stop_reason === 'max_tokens') throw new Error('the draft ran past max_tokens'); - return message; -} - -function replyText(message: Anthropic.Beta.BetaMessage): string { - return message.content.flatMap(block => (block.type === 'text' ? [block.text] : [])).join('\n'); -} - async function main(): Promise { const slug = arg('slug'); const urls = args('docs').slice(0, MAX_DOCS); @@ -99,23 +81,19 @@ async function main(): Promise { const docs = await Promise.all(urls.map(fetchDoc)); console.log(`Read ${docs.length} page(s): ${docs.map(d => `${d.url} (${d.text.length} chars)`).join(', ')}`); - const client = new Anthropic(); - const messages: Anthropic.Beta.BetaMessageParam[] = [{ - role: 'user', - content: [ - ...docs.map(doc => ({ type: 'text' as const, text: `\n${doc.text}\n` })), - { type: 'text' as const, text: 'Write the spec for this API from these documents.' } - ] - }]; + const session = draftSession(chooseProvider(arg('provider')), SYSTEM, { model: arg('model') }); + console.log(`Drafting with ${session.provider} (${session.model}).`); + let turn = [ + ...docs.map(doc => `\n${doc.text}\n`), + 'Write the spec for this API from these documents.' + ]; let spec: unknown = null; let problems: string[] = []; for (let round = 0; round <= REPAIR_ROUNDS; round++) { - const reply = await ask(client, messages); - // Keep the whole reply, thinking included, so the next turn continues the same conversation. - messages.push({ role: 'assistant', content: reply.content }); + const reply = await session.send(turn); try { - spec = jsonObjectIn(replyText(reply)); + spec = jsonObjectIn(reply); problems = checkDraft(spec); } catch (err) { spec = null; @@ -123,7 +101,7 @@ async function main(): Promise { } console.log(`round ${round + 1}: ${problems.length === 0 ? 'passes the checks' : `${problems.length} problem(s)`}`); if (problems.length === 0 || round === REPAIR_ROUNDS) break; - messages.push({ role: 'user', content: `The spec has these problems. Fix them using the documents, then output the whole corrected spec:\n- ${problems.join('\n- ')}` }); + turn = [`The spec has these problems. Fix them using the documents, then output the whole corrected spec:\n- ${problems.join('\n- ')}`]; } const live = spec && problems.length === 0 ? await verifySpec(`draft:${slug}`, spec, `omni:drafts/${slug}`, keyEnv) : []; @@ -135,7 +113,7 @@ async function main(): Promise { const report = [ `# Draft spec: ${slug}`, '', - `Drafted ${new Date().toISOString()} by ${MODEL} from:`, + `Drafted ${new Date().toISOString()} by ${session.provider} ${session.model} from:`, ...docs.map(d => `- ${d.url}`), '', '## Mechanical checks', @@ -158,7 +136,8 @@ async function main(): Promise { main().catch((err: unknown) => { const noCredentials = err instanceof Error && /resolve authentication method/i.test(err.message); - if (err instanceof Anthropic.AuthenticationError || noCredentials) console.error('forge:spec: no valid Anthropic credentials (set ANTHROPIC_API_KEY in .env, or run `ant auth login`).'); + if (err instanceof Anthropic.AuthenticationError || noCredentials) console.error('forge:spec: no valid Anthropic credentials (set ANTHROPIC_API_KEY in .env, run `ant auth login`, or pick --provider openai|google).'); + else if (err instanceof DraftRefused) console.error(`forge:spec: ${err.message}`); else if (err instanceof Anthropic.RateLimitError) console.error('forge:spec: rate limited by the Claude API; try again shortly.'); else if (err instanceof Anthropic.APIError) console.error(`forge:spec: Claude API error ${err.status}: ${err.message}`); else console.error('forge:spec failed:', err instanceof Error ? err.message : err); diff --git a/scripts/lib/drafters.ts b/scripts/lib/drafters.ts new file mode 100644 index 0000000..f7df533 --- /dev/null +++ b/scripts/lib/drafters.ts @@ -0,0 +1,192 @@ +// The model behind the spec forge (scripts/forge-spec.ts): Claude, OpenAI or +// Gemini. Each is a session that keeps its own conversation in its native +// shape, so a repair round continues the same conversation. The key is read +// from the environment and sent only in the request's own header; no error +// message here repeats a response body. + +import Anthropic from '@anthropic-ai/sdk'; + +export type DraftProvider = 'anthropic' | 'openai' | 'google'; + +export const DEFAULT_MODELS: Record = { + anthropic: 'claude-opus-5-5', + openai: 'gpt-5', + google: 'gemini-2.5-pro' +}; + +const KEY_ENV: Record, string> = { + openai: 'OPENAI_API_KEY', + google: 'GOOGLE_API_KEY' +}; + +const MAX_OUTPUT_TOKENS = 64_000; +const REQUEST_TIMEOUT_MS = 10 * 60_000; + +export interface DraftSession { + provider: DraftProvider; + model: string; + /** Send one user turn (the first carries the documents) and return the reply's text. */ + send(userText: string[]): Promise; +} + +/** The provider asked for, or the first one with a key: Claude, then OpenAI, then Gemini. */ +export function chooseProvider(requested: string | undefined, env: Record = process.env): DraftProvider { + if (requested !== undefined) { + if (requested === 'anthropic' || requested === 'openai' || requested === 'google') return requested; + throw new Error(`--provider must be anthropic, openai or google, not ${requested}`); + } + if (env.ANTHROPIC_API_KEY?.trim()) return 'anthropic'; + if (env.OPENAI_API_KEY?.trim()) return 'openai'; + if (env.GOOGLE_API_KEY?.trim()) return 'google'; + // No key at all: Claude, which can also use an `ant auth login` profile. + return 'anthropic'; +} + +export class DraftRefused extends Error {} + +// ---- reading replies (pure, so they are tested without a network) ---------- + +/** The text of an OpenAI chat completion, or why there is none. */ +export function openAiReply(body: unknown): string { + const choice = (body as { choices?: { finish_reason?: string; message?: { content?: unknown; refusal?: unknown } }[] }).choices?.[0]; + if (!choice?.message) throw new Error('OpenAI sent no reply'); + if (typeof choice.message.refusal === 'string' && choice.message.refusal) throw new DraftRefused('OpenAI declined to draft this spec'); + if (choice.finish_reason === 'length') throw new Error('the draft ran past the output limit'); + if (choice.finish_reason === 'content_filter') throw new DraftRefused('OpenAI filtered the draft'); + if (typeof choice.message.content !== 'string' || !choice.message.content) throw new Error('OpenAI sent an empty reply'); + return choice.message.content; +} + +/** The text of a Gemini reply (thought parts left out), or why there is none. */ +export function geminiReply(body: unknown): string { + const value = body as { + promptFeedback?: { blockReason?: string }; + candidates?: { finishReason?: string; content?: { parts?: { text?: unknown; thought?: unknown }[] } }[]; + }; + if (value.promptFeedback?.blockReason) throw new DraftRefused(`Gemini blocked the request (${value.promptFeedback.blockReason})`); + const candidate = value.candidates?.[0]; + if (!candidate) throw new Error('Gemini sent no reply'); + if (candidate.finishReason === 'MAX_TOKENS') throw new Error('the draft ran past the output limit'); + if (candidate.finishReason === 'SAFETY' || candidate.finishReason === 'RECITATION' || candidate.finishReason === 'PROHIBITED_CONTENT') { + throw new DraftRefused(`Gemini stopped the draft (${candidate.finishReason})`); + } + const text = (candidate.content?.parts ?? []) + .filter(part => part.thought !== true && typeof part.text === 'string') + .map(part => part.text as string) + .join('\n'); + if (!text) throw new Error('Gemini sent an empty reply'); + return text; +} + +/** An HTTP failure as a short line: the status and the provider's own error message, never the body. */ +async function httpFailure(provider: string, response: Response): Promise { + let message = ''; + try { + const body = await response.json() as { error?: { message?: unknown } }; + if (typeof body.error?.message === 'string') message = `: ${body.error.message.slice(0, 200)}`; + } catch { + // Not JSON: the status says enough. + } + return new Error(`${provider} API error ${response.status}${message}`); +} + +function keyFor(provider: Exclude, env: Record): string { + const key = env[KEY_ENV[provider]]?.trim(); + if (!key) throw new Error(`no ${provider === 'openai' ? 'OpenAI' : 'Gemini'} key (set ${KEY_ENV[provider]} in .env)`); + return key; +} + +// ---- sessions --------------------------------------------------------------- + +function anthropicSession(system: string, model: string): DraftSession { + const client = new Anthropic(); + const messages: Anthropic.Beta.BetaMessageParam[] = []; + return { + provider: 'anthropic', + model, + async send(userText) { + messages.push({ role: 'user', content: userText.map(text => ({ type: 'text' as const, text })) }); + const stream = client.beta.messages.stream({ + model, + max_tokens: MAX_OUTPUT_TOKENS, + system, + thinking: { type: 'adaptive' }, + output_config: { effort: 'high' }, + // On a policy decline, re-run on a suitable model instead of stopping. + betas: ['server-side-fallback-2026-07-01'], + fallbacks: 'default', + messages + } as Parameters[0]); + const message = await stream.finalMessage(); + if (message.stop_reason === 'refusal') throw new DraftRefused('Claude declined to draft this spec'); + if (message.stop_reason === 'max_tokens') throw new Error('the draft ran past max_tokens'); + // Keep the whole reply, thinking included, so the next turn continues the same conversation. + messages.push({ role: 'assistant', content: message.content }); + return message.content.flatMap(block => (block.type === 'text' ? [block.text] : [])).join('\n'); + } + }; +} + +function openAiSession(system: string, model: string, env: Record, fetchImpl: typeof fetch): DraftSession { + const key = keyFor('openai', env); + const messages: { role: 'system' | 'user' | 'assistant'; content: string }[] = [{ role: 'system', content: system }]; + return { + provider: 'openai', + model, + async send(userText) { + messages.push({ role: 'user', content: userText.join('\n\n') }); + const response = await fetchImpl('https://api.openai.com/v1/chat/completions', { + method: 'POST', + redirect: 'error', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, + body: JSON.stringify({ model, messages, max_completion_tokens: MAX_OUTPUT_TOKENS }) + }); + if (!response.ok) throw await httpFailure('OpenAI', response); + const text = openAiReply(await response.json()); + messages.push({ role: 'assistant', content: text }); + return text; + } + }; +} + +function geminiSession(system: string, model: string, env: Record, fetchImpl: typeof fetch): DraftSession { + const key = keyFor('google', env); + if (!/^[a-z0-9.-]{1,64}$/i.test(model)) throw new Error('--model is not a Gemini model name'); + const contents: { role: 'user' | 'model'; parts: { text: string }[] }[] = []; + return { + provider: 'google', + model, + async send(userText) { + contents.push({ role: 'user', parts: userText.map(text => ({ text })) }); + const response = await fetchImpl(`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`, { + method: 'POST', + redirect: 'error', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + headers: { 'content-type': 'application/json', 'x-goog-api-key': key }, + body: JSON.stringify({ + systemInstruction: { parts: [{ text: system }] }, + contents, + generationConfig: { maxOutputTokens: MAX_OUTPUT_TOKENS, responseMimeType: 'application/json' } + }) + }); + if (!response.ok) throw await httpFailure('Gemini', response); + const text = geminiReply(await response.json()); + contents.push({ role: 'model', parts: [{ text }] }); + return text; + } + }; +} + +export function draftSession( + provider: DraftProvider, + system: string, + options: { model?: string; env?: Record; fetch?: typeof fetch } = {} +): DraftSession { + const model = options.model ?? DEFAULT_MODELS[provider]; + const env = options.env ?? process.env; + const fetchImpl = options.fetch ?? fetch; + if (provider === 'openai') return openAiSession(system, model, env, fetchImpl); + if (provider === 'google') return geminiSession(system, model, env, fetchImpl); + return anthropicSession(system, model); +} diff --git a/src/core/capabilities/specDraftProviders.test.ts b/src/core/capabilities/specDraftProviders.test.ts new file mode 100644 index 0000000..dd47d9c --- /dev/null +++ b/src/core/capabilities/specDraftProviders.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it, vi } from 'vitest'; +import { chooseProvider, draftSession, DraftRefused, geminiReply, openAiReply } from '../../../scripts/lib/drafters'; + +const OPENAI_KEY = 'sk-test-openai-0123456789'; +const GOOGLE_KEY = 'AIza-test-google-0123456789'; + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }); +} + +describe('which model drafts the spec', () => { + it('takes --provider, else the first key present: Claude, OpenAI, Gemini', () => { + expect(chooseProvider('google', { ANTHROPIC_API_KEY: 'x' })).toBe('google'); + expect(chooseProvider(undefined, { ANTHROPIC_API_KEY: 'x', OPENAI_API_KEY: 'y' })).toBe('anthropic'); + expect(chooseProvider(undefined, { OPENAI_API_KEY: 'y', GOOGLE_API_KEY: 'z' })).toBe('openai'); + expect(chooseProvider(undefined, { GOOGLE_API_KEY: 'z' })).toBe('google'); + expect(chooseProvider(undefined, {})).toBe('anthropic'); + expect(() => chooseProvider('deepseek', {})).toThrow(/anthropic, openai or google/); + }); +}); + +describe('reading replies', () => { + it('OpenAI: the text, a refusal, or a cut-off draft', () => { + expect(openAiReply({ choices: [{ finish_reason: 'stop', message: { content: '{"openapi":"3.0.3"}' } }] })).toBe('{"openapi":"3.0.3"}'); + expect(() => openAiReply({ choices: [{ message: { content: null, refusal: 'no' } }] })).toThrow(DraftRefused); + expect(() => openAiReply({ choices: [{ finish_reason: 'length', message: { content: '{' } }] })).toThrow(/output limit/); + }); + + it('Gemini: the text without thought parts, a block, or a cut-off draft', () => { + expect(geminiReply({ candidates: [{ finishReason: 'STOP', content: { parts: [{ text: 'thinking…', thought: true }, { text: '{"a":1}' }] } }] })).toBe('{"a":1}'); + expect(() => geminiReply({ promptFeedback: { blockReason: 'SAFETY' } })).toThrow(DraftRefused); + expect(() => geminiReply({ candidates: [{ finishReason: 'MAX_TOKENS', content: { parts: [{ text: '{' }] } }] })).toThrow(/output limit/); + }); +}); + +describe('OpenAI session', () => { + it('sends the key only as a bearer header, and keeps the conversation across repair rounds', async () => { + const fetchMock = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => jsonResponse({ choices: [{ finish_reason: 'stop', message: { content: '{"draft":true}' } }] })); + const session = draftSession('openai', 'SYSTEM', { env: { OPENAI_API_KEY: OPENAI_KEY }, fetch: fetchMock as unknown as typeof fetch }); + expect(session.model).toBe('gpt-5'); + await session.send(['docs', 'Write the spec.']); + await session.send(['Fix these problems.']); + const [url, init] = fetchMock.mock.calls[1]; + expect(String(url)).toBe('https://api.openai.com/v1/chat/completions'); + expect((init?.headers as Record).authorization).toBe(`Bearer ${OPENAI_KEY}`); + const body = JSON.parse(String(init?.body)) as { model: string; messages: { role: string; content: string }[] }; + expect(body.model).toBe('gpt-5'); + expect(body.messages.map(m => m.role)).toEqual(['system', 'user', 'assistant', 'user']); + expect(body.messages[2].content).toBe('{"draft":true}'); + expect(String(init?.body)).not.toContain(OPENAI_KEY); + }); + + it('reports an HTTP error by status and the provider\'s message, not the body', async () => { + const fetchMock = vi.fn(async () => jsonResponse({ error: { message: 'Incorrect API key provided' }, debug: 'internal' }, 401)); + const session = draftSession('openai', 'SYSTEM', { env: { OPENAI_API_KEY: OPENAI_KEY }, fetch: fetchMock as unknown as typeof fetch }); + await expect(session.send(['x'])).rejects.toThrow('OpenAI API error 401: Incorrect API key provided'); + }); + + it('says which variable to set when there is no key', () => { + expect(() => draftSession('openai', 'SYSTEM', { env: {} })).toThrow('no OpenAI key (set OPENAI_API_KEY in .env)'); + }); +}); + +describe('Gemini session', () => { + it('sends the key in a header, never the URL, and asks for JSON', async () => { + const fetchMock = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => jsonResponse({ candidates: [{ finishReason: 'STOP', content: { parts: [{ text: '{"draft":true}' }] } }] })); + const session = draftSession('google', 'SYSTEM', { env: { GOOGLE_API_KEY: GOOGLE_KEY }, fetch: fetchMock as unknown as typeof fetch }); + expect(await session.send(['docs'])).toBe('{"draft":true}'); + const [url, init] = fetchMock.mock.calls[0]; + expect(String(url)).toBe('https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent'); + expect(String(url)).not.toContain(GOOGLE_KEY); + expect((init?.headers as Record)['x-goog-api-key']).toBe(GOOGLE_KEY); + const body = JSON.parse(String(init?.body)) as { systemInstruction: unknown; generationConfig: { responseMimeType: string } }; + expect(body.systemInstruction).toEqual({ parts: [{ text: 'SYSTEM' }] }); + expect(body.generationConfig.responseMimeType).toBe('application/json'); + }); + + it('refuses a model name that could change the URL path', () => { + expect(() => draftSession('google', 'SYSTEM', { model: '../../v1/files', env: { GOOGLE_API_KEY: GOOGLE_KEY } })).toThrow(/not a Gemini model name/); + }); +});