Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions APEX_PLAN.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@ this order:
first run always works.
6. **Spec forge.** A developer tool that drafts a curated spec from provider
docs, checks it against live calls, and leaves a person to approve it.
Status: drafts with Claude, OpenAI or Gemini (`--provider`). Measured once
on 2026-10-12 with Gemini 2.5 Pro: Open Library search docs, passed the
checks in round 1, both operations answered live. Claude and OpenAI are
tested with stubs only.
7. **Data plant.** Server-side house keys, used only where a provider's terms
allow an application key, behind a shared cache with per-user quotas and
provenance on every value. Lifts the "broad provider catalog" cut in
Expand Down
59 changes: 19 additions & 40 deletions scripts/forge-spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// FLYSPACE: SPEC FORGE
//
// npm run forge:spec -- --slug openlibrary --docs https://openlibrary.org/dev/docs/api/search [--docs <url> ...] [--key-env NAME]
// [--provider anthropic|openai|google] [--model NAME]
//
// APEX_PLAN 6.6. Drafts a curated OpenAPI spec from a provider's own docs,
// checks it mechanically (specDraft.ts: compiles, reads only, every required
Expand All @@ -15,17 +16,20 @@
// It never registers anything. A person reads the draft against the docs,
// then adds it to curatedApis.ts by hand; that review is the approval.
//
// Needs Anthropic credentials (ANTHROPIC_API_KEY, or an `ant auth login`
// profile). The fetched docs are sent to Claude as data; nothing else is.
// The draft comes from Claude (ANTHROPIC_API_KEY, or an `ant auth login`
// profile), OpenAI (OPENAI_API_KEY) or Gemini (GOOGLE_API_KEY): --provider
// picks one, otherwise the first with a key, in that order (scripts/lib/drafters.ts).
// The fetched docs are sent to that provider as data; nothing else is.
// Whatever drafts it, the same checks and live calls decide whether it passes.
// ============================================

import { mkdirSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import Anthropic from '@anthropic-ai/sdk';
import { checkDraft, docsText, jsonObjectIn } from '../src/core/capabilities/specDraft';
import { chooseProvider, draftSession, DraftRefused } from './lib/drafters';
import { verifySpec } from './lib/verifySpec';

const MODEL = 'claude-opus-5-5';
const MAX_DOC_CHARS = 150_000;
const MAX_DOCS = 5;
const REPAIR_ROUNDS = 2;
Expand Down Expand Up @@ -66,28 +70,6 @@ async function fetchDoc(url: string): Promise<{ url: string; text: string }> {
return { url, text: text.slice(0, MAX_DOC_CHARS) };
}

async function ask(client: Anthropic, messages: Anthropic.Beta.BetaMessageParam[]): Promise<Anthropic.Beta.BetaMessage> {
const stream = client.beta.messages.stream({
model: MODEL,
max_tokens: 64_000,
system: SYSTEM,
thinking: { type: 'adaptive' },
output_config: { effort: 'high' },
// On a policy decline, re-run on a suitable model instead of stopping.
betas: ['server-side-fallback-2026-07-01'],
fallbacks: 'default',
messages
} as Parameters<typeof client.beta.messages.stream>[0]);
const message = await stream.finalMessage();
if (message.stop_reason === 'refusal') throw new Error('Claude declined to draft this spec');
if (message.stop_reason === 'max_tokens') throw new Error('the draft ran past max_tokens');
return message;
}

function replyText(message: Anthropic.Beta.BetaMessage): string {
return message.content.flatMap(block => (block.type === 'text' ? [block.text] : [])).join('\n');
}

async function main(): Promise<void> {
const slug = arg('slug');
const urls = args('docs').slice(0, MAX_DOCS);
Expand All @@ -99,31 +81,27 @@ async function main(): Promise<void> {
const docs = await Promise.all(urls.map(fetchDoc));
console.log(`Read ${docs.length} page(s): ${docs.map(d => `${d.url} (${d.text.length} chars)`).join(', ')}`);

const client = new Anthropic();
const messages: Anthropic.Beta.BetaMessageParam[] = [{
role: 'user',
content: [
...docs.map(doc => ({ type: 'text' as const, text: `<document source="${doc.url}">\n${doc.text}\n</document>` })),
{ type: 'text' as const, text: 'Write the spec for this API from these documents.' }
]
}];
const session = draftSession(chooseProvider(arg('provider')), SYSTEM, { model: arg('model') });
console.log(`Drafting with ${session.provider} (${session.model}).`);
let turn = [
...docs.map(doc => `<document source="${doc.url}">\n${doc.text}\n</document>`),
'Write the spec for this API from these documents.'
];

let spec: unknown = null;
let problems: string[] = [];
for (let round = 0; round <= REPAIR_ROUNDS; round++) {
const reply = await ask(client, messages);
// Keep the whole reply, thinking included, so the next turn continues the same conversation.
messages.push({ role: 'assistant', content: reply.content });
const reply = await session.send(turn);
try {
spec = jsonObjectIn(replyText(reply));
spec = jsonObjectIn(reply);
problems = checkDraft(spec);
} catch (err) {
spec = null;
problems = [err instanceof Error ? err.message : String(err)];
}
console.log(`round ${round + 1}: ${problems.length === 0 ? 'passes the checks' : `${problems.length} problem(s)`}`);
if (problems.length === 0 || round === REPAIR_ROUNDS) break;
messages.push({ role: 'user', content: `The spec has these problems. Fix them using the documents, then output the whole corrected spec:\n- ${problems.join('\n- ')}` });
turn = [`The spec has these problems. Fix them using the documents, then output the whole corrected spec:\n- ${problems.join('\n- ')}`];
}

const live = spec && problems.length === 0 ? await verifySpec(`draft:${slug}`, spec, `omni:drafts/${slug}`, keyEnv) : [];
Expand All @@ -135,7 +113,7 @@ async function main(): Promise<void> {
const report = [
`# Draft spec: ${slug}`,
'',
`Drafted ${new Date().toISOString()} by ${MODEL} from:`,
`Drafted ${new Date().toISOString()} by ${session.provider} ${session.model} from:`,
...docs.map(d => `- ${d.url}`),
'',
'## Mechanical checks',
Expand All @@ -158,7 +136,8 @@ async function main(): Promise<void> {

main().catch((err: unknown) => {
const noCredentials = err instanceof Error && /resolve authentication method/i.test(err.message);
if (err instanceof Anthropic.AuthenticationError || noCredentials) console.error('forge:spec: no valid Anthropic credentials (set ANTHROPIC_API_KEY in .env, or run `ant auth login`).');
if (err instanceof Anthropic.AuthenticationError || noCredentials) console.error('forge:spec: no valid Anthropic credentials (set ANTHROPIC_API_KEY in .env, run `ant auth login`, or pick --provider openai|google).');
else if (err instanceof DraftRefused) console.error(`forge:spec: ${err.message}`);
else if (err instanceof Anthropic.RateLimitError) console.error('forge:spec: rate limited by the Claude API; try again shortly.');
else if (err instanceof Anthropic.APIError) console.error(`forge:spec: Claude API error ${err.status}: ${err.message}`);
else console.error('forge:spec failed:', err instanceof Error ? err.message : err);
Expand Down
192 changes: 192 additions & 0 deletions scripts/lib/drafters.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
// The model behind the spec forge (scripts/forge-spec.ts): Claude, OpenAI or
// Gemini. Each is a session that keeps its own conversation in its native
// shape, so a repair round continues the same conversation. The key is read
// from the environment and sent only in the request's own header; no error
// message here repeats a response body.

import Anthropic from '@anthropic-ai/sdk';

export type DraftProvider = 'anthropic' | 'openai' | 'google';

export const DEFAULT_MODELS: Record<DraftProvider, string> = {
anthropic: 'claude-opus-5-5',
openai: 'gpt-5',
google: 'gemini-2.5-pro'
};

const KEY_ENV: Record<Exclude<DraftProvider, 'anthropic'>, string> = {
openai: 'OPENAI_API_KEY',
google: 'GOOGLE_API_KEY'
};

const MAX_OUTPUT_TOKENS = 64_000;
const REQUEST_TIMEOUT_MS = 10 * 60_000;

export interface DraftSession {
provider: DraftProvider;
model: string;
/** Send one user turn (the first carries the documents) and return the reply's text. */
send(userText: string[]): Promise<string>;
}

/** The provider asked for, or the first one with a key: Claude, then OpenAI, then Gemini. */
export function chooseProvider(requested: string | undefined, env: Record<string, string | undefined> = process.env): DraftProvider {
if (requested !== undefined) {
if (requested === 'anthropic' || requested === 'openai' || requested === 'google') return requested;
throw new Error(`--provider must be anthropic, openai or google, not ${requested}`);
}
if (env.ANTHROPIC_API_KEY?.trim()) return 'anthropic';
if (env.OPENAI_API_KEY?.trim()) return 'openai';
if (env.GOOGLE_API_KEY?.trim()) return 'google';
// No key at all: Claude, which can also use an `ant auth login` profile.
return 'anthropic';
}

export class DraftRefused extends Error {}

// ---- reading replies (pure, so they are tested without a network) ----------

/** The text of an OpenAI chat completion, or why there is none. */
export function openAiReply(body: unknown): string {
const choice = (body as { choices?: { finish_reason?: string; message?: { content?: unknown; refusal?: unknown } }[] }).choices?.[0];
if (!choice?.message) throw new Error('OpenAI sent no reply');
if (typeof choice.message.refusal === 'string' && choice.message.refusal) throw new DraftRefused('OpenAI declined to draft this spec');
if (choice.finish_reason === 'length') throw new Error('the draft ran past the output limit');
if (choice.finish_reason === 'content_filter') throw new DraftRefused('OpenAI filtered the draft');
if (typeof choice.message.content !== 'string' || !choice.message.content) throw new Error('OpenAI sent an empty reply');
return choice.message.content;
}

/** The text of a Gemini reply (thought parts left out), or why there is none. */
export function geminiReply(body: unknown): string {
const value = body as {
promptFeedback?: { blockReason?: string };
candidates?: { finishReason?: string; content?: { parts?: { text?: unknown; thought?: unknown }[] } }[];
};
if (value.promptFeedback?.blockReason) throw new DraftRefused(`Gemini blocked the request (${value.promptFeedback.blockReason})`);
const candidate = value.candidates?.[0];
if (!candidate) throw new Error('Gemini sent no reply');
if (candidate.finishReason === 'MAX_TOKENS') throw new Error('the draft ran past the output limit');
if (candidate.finishReason === 'SAFETY' || candidate.finishReason === 'RECITATION' || candidate.finishReason === 'PROHIBITED_CONTENT') {
throw new DraftRefused(`Gemini stopped the draft (${candidate.finishReason})`);
}
const text = (candidate.content?.parts ?? [])
.filter(part => part.thought !== true && typeof part.text === 'string')
.map(part => part.text as string)
.join('\n');
if (!text) throw new Error('Gemini sent an empty reply');
return text;
}

/** An HTTP failure as a short line: the status and the provider's own error message, never the body. */
async function httpFailure(provider: string, response: Response): Promise<Error> {
let message = '';
try {
const body = await response.json() as { error?: { message?: unknown } };
if (typeof body.error?.message === 'string') message = `: ${body.error.message.slice(0, 200)}`;

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Prevent provider errors from logging API keys

When OpenAI rejects an invalid or revoked key, its documented 401 message includes the submitted (masked) key; this helper copies up to 200 characters of that message into the thrown error, which forge-spec.ts then writes to stderr. Running the forge with a bad OPENAI_API_KEY therefore puts a value derived from that secret into CI or shell logs; report only the HTTP status (or scrub against the configured key) instead. OpenAI’s documented incorrect-key response

AGENTS.md reference: AGENTS.md:L102-L103

Useful? React with 👍 / 👎.

} catch {
// Not JSON: the status says enough.
}
return new Error(`${provider} API error ${response.status}${message}`);
}

function keyFor(provider: Exclude<DraftProvider, 'anthropic'>, env: Record<string, string | undefined>): string {
const key = env[KEY_ENV[provider]]?.trim();
if (!key) throw new Error(`no ${provider === 'openai' ? 'OpenAI' : 'Gemini'} key (set ${KEY_ENV[provider]} in .env)`);
return key;
}

// ---- sessions ---------------------------------------------------------------

function anthropicSession(system: string, model: string): DraftSession {
const client = new Anthropic();
const messages: Anthropic.Beta.BetaMessageParam[] = [];
return {
provider: 'anthropic',
model,
async send(userText) {
messages.push({ role: 'user', content: userText.map(text => ({ type: 'text' as const, text })) });
const stream = client.beta.messages.stream({
model,
max_tokens: MAX_OUTPUT_TOKENS,
system,
thinking: { type: 'adaptive' },
output_config: { effort: 'high' },
// On a policy decline, re-run on a suitable model instead of stopping.
betas: ['server-side-fallback-2026-07-01'],
fallbacks: 'default',
messages
} as Parameters<typeof client.beta.messages.stream>[0]);
const message = await stream.finalMessage();
if (message.stop_reason === 'refusal') throw new DraftRefused('Claude declined to draft this spec');
if (message.stop_reason === 'max_tokens') throw new Error('the draft ran past max_tokens');
// Keep the whole reply, thinking included, so the next turn continues the same conversation.
messages.push({ role: 'assistant', content: message.content });
return message.content.flatMap(block => (block.type === 'text' ? [block.text] : [])).join('\n');
}
};
}

function openAiSession(system: string, model: string, env: Record<string, string | undefined>, fetchImpl: typeof fetch): DraftSession {
const key = keyFor('openai', env);
const messages: { role: 'system' | 'user' | 'assistant'; content: string }[] = [{ role: 'system', content: system }];
return {
provider: 'openai',
model,
async send(userText) {
messages.push({ role: 'user', content: userText.join('\n\n') });
const response = await fetchImpl('https://api.openai.com/v1/chat/completions', {
method: 'POST',
redirect: 'error',
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
body: JSON.stringify({ model, messages, max_completion_tokens: MAX_OUTPUT_TOKENS })
});
if (!response.ok) throw await httpFailure('OpenAI', response);
const text = openAiReply(await response.json());
messages.push({ role: 'assistant', content: text });
return text;
}
};
}

function geminiSession(system: string, model: string, env: Record<string, string | undefined>, fetchImpl: typeof fetch): DraftSession {
const key = keyFor('google', env);
if (!/^[a-z0-9.-]{1,64}$/i.test(model)) throw new Error('--model is not a Gemini model name');
const contents: { role: 'user' | 'model'; parts: { text: string }[] }[] = [];
return {
provider: 'google',
model,
async send(userText) {
contents.push({ role: 'user', parts: userText.map(text => ({ text })) });
const response = await fetchImpl(`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`, {
method: 'POST',
redirect: 'error',
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
headers: { 'content-type': 'application/json', 'x-goog-api-key': key },
body: JSON.stringify({
systemInstruction: { parts: [{ text: system }] },
contents,
generationConfig: { maxOutputTokens: MAX_OUTPUT_TOKENS, responseMimeType: 'application/json' }
})
});
if (!response.ok) throw await httpFailure('Gemini', response);
const text = geminiReply(await response.json());
contents.push({ role: 'model', parts: [{ text }] });
return text;
}
};
}

export function draftSession(
provider: DraftProvider,
system: string,
options: { model?: string; env?: Record<string, string | undefined>; fetch?: typeof fetch } = {}
): DraftSession {
const model = options.model ?? DEFAULT_MODELS[provider];
const env = options.env ?? process.env;
const fetchImpl = options.fetch ?? fetch;
if (provider === 'openai') return openAiSession(system, model, env, fetchImpl);
if (provider === 'google') return geminiSession(system, model, env, fetchImpl);
return anthropicSession(system, model);
}
Loading
Loading