From 1b5d144f4f89657e626455fc308fcff8aa863886 Mon Sep 17 00:00:00 2001 From: CMaintz Date: Tue, 29 Sep 2026 17:24:23 +0200 Subject: [PATCH 1/2] feat(jev): advisory Jev layer v0.1 - shared client + routing core First slice of the foundry-jev-integration spec: the reusable, deterministic core that later phases (review pre-filter, tool-call triage) build on. - scripts/jev/client.mjs: JevProvider port lifted from jev-triage (TypeSafe + Cloudflare adapters, postJson with 429/529 backoff) + providerFromEnv, which returns null with no key so every caller fails open. Honors TYPESAFE_AI_BASE_URL. - scripts/jev/route.mjs: pure routing/triage core (routeReview, triageToolcall, question templates). Fail-open invariant: a missing or low-confidence answer always widens review, never narrows it. - scripts/jev/boundary.test.mjs: mechanical guard for the spec's core invariant - reds the build if any CI workflow or mise gate verb references scripts/jev, so a ~68%-accurate model can never reach the deterministic oracle. - 18 tests via node:test (zero deps). docs/FEATURES.md + module README. Advisory only, opt-in via JEV_API_KEY, no change to any gate. Later: review depth/lens pre-filter and per-turn tool-call triage in cmaintz-skills. --- docs/FEATURES.md | 1 + scripts/jev/README.md | 52 ++++++++++++++++++ scripts/jev/boundary.test.mjs | 35 +++++++++++++ scripts/jev/client.mjs | 81 ++++++++++++++++++++++++++++ scripts/jev/client.test.mjs | 83 +++++++++++++++++++++++++++++ scripts/jev/route.mjs | 99 +++++++++++++++++++++++++++++++++++ scripts/jev/route.test.mjs | 64 ++++++++++++++++++++++ 7 files changed, 415 insertions(+) create mode 100644 scripts/jev/README.md create mode 100644 scripts/jev/boundary.test.mjs create mode 100644 scripts/jev/client.mjs create mode 100644 scripts/jev/client.test.mjs create mode 100644 scripts/jev/route.mjs create mode 100644 scripts/jev/route.test.mjs diff --git a/docs/FEATURES.md b/docs/FEATURES.md index f5db137..36ab12f 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -75,6 +75,7 @@ fails the verb instead of going unchecked. - `npm-audit-ratchet.mjs` — ratcheted `npm audit` for the ts `audit` verb: fails on any critical not in `.audit-allowlist.json` and on stale entries, so accepted CVE debt can only shrink (npm audit has no native per-advisory ignore). Degrades to plain `npm audit --audit-level=critical` with no allowlist. Reads the report from stdin. - `foundry-init.sh` — one-shot repo scaffold. - `setup-labels.sh` — create the GitHub labels the workflows + ticket state machine need (agent:ready/working/blocked, align, ruleset-change, autofix). Idempotent; run by `foundry-init`. +- `jev/` — the **advisory** Jev layer (`client.mjs` provider port lifted from jev-triage; `route.mjs` pure routing/triage core). Near-free System-One decisions on the **proposer** side only: which diff hunks warrant deep review and on which lens, and whether a turn looks destructive. Opt-in via `JEV_API_KEY` (+ `JEV_PROVIDER`/`JEV_MODEL`/`TYPESAFE_AI_BASE_URL`); absent key ⇒ every caller fails open to current behavior. **Never in the deterministic gate** — `boundary.test.mjs` reds the build if any workflow or mise verb references it. Tests: `node --test scripts/jev/*.test.mjs`. ## Agent half — `cmaintz-skills` diff --git a/scripts/jev/README.md b/scripts/jev/README.md new file mode 100644 index 0000000..62af165 --- /dev/null +++ b/scripts/jev/README.md @@ -0,0 +1,52 @@ +# scripts/jev - Foundry's advisory Jev layer + +Near-free [Jev](https://typesafe.ai) (TypeSafe System One) decisions for the +**proposer** side of Foundry: which diff hunks deserve deep review and on which +lens, and whether a turn looks destructive. This is where Jev earns its place - +upstream of the expensive LLM, never upstream of the deterministic oracle. + +## The one rule + +Jev is ~68% accurate, so it **never** sits in the authoritative gate +(`lint` / `typecheck` / `test` / `audit`). It only routes and triages: a wrong +call costs a missed shortcut, never a false pass. `boundary.test.mjs` fails the +build if any CI workflow or mise gate verb ever references this directory. + +## Pieces + +- `client.mjs` - the `JevProvider` port (`TypeSafeProvider`, `CloudflareProvider`, + `postJson` with 429/529 backoff) plus `providerFromEnv`, which returns `null` + when no key is set so every caller fails open to current behavior. +- `route.mjs` - pure, no-network core: `routeReview(answers, cfg)`, + `triageToolcall(answer, cfg)`, and the question templates. Fail-open is the + invariant: a missing or low-confidence answer always widens review, never narrows it. + +## Usage + +```js +import { providerFromEnv } from './scripts/jev/client.mjs'; +import { reviewQuestions, routeReview } from './scripts/jev/route.mjs'; + +const jev = providerFromEnv(); +if (!jev) return; // no key: skip Jev, review everything as before + +const { answers } = await jev.evaluate({ + state: { file, hunk, ticket }, + questions: reviewQuestions(), +}); +const { review, lens, reason } = routeReview(answers); +// review only the above-threshold hunks, on `lens`; log the rest (still get baseline review). +``` + +## Config + +Read from the environment (keys never in CI): +`JEV_API_KEY`, `JEV_PROVIDER` (`typesafe` | `cloudflare`), `JEV_MODEL` +(defaults to `jev-latest`), `CLOUDFLARE_ACCOUNT_ID`, and `TYPESAFE_AI_BASE_URL` +(point the direct call at a self-host, proxy, or mock). + +## Tests + +``` +node --test scripts/jev/*.test.mjs +``` diff --git a/scripts/jev/boundary.test.mjs b/scripts/jev/boundary.test.mjs new file mode 100644 index 0000000..45636ac --- /dev/null +++ b/scripts/jev/boundary.test.mjs @@ -0,0 +1,35 @@ +import assert from 'node:assert/strict'; +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { test } from 'node:test'; + +// The invariant (spec foundry-jev-integration, deterministic oracle / probabilistic +// proposer): Jev is advisory-only and must NEVER be referenced by a gate-defining +// file - not a CI workflow, not a mise gate verb. A reference here would mean a +// ~68%-accurate model crept into the authoritative pass/fail path. This test makes +// that a red build rather than a code-review hope. Run from the repo root. +const BANNED = 'scripts/jev'; + +function filesIn(dir, ext) { + try { + return readdirSync(dir) + .filter((name) => name.endsWith(ext)) + .map((name) => join(dir, name)); + } catch { + return []; + } +} + +function assertNoneReference(files, why) { + for (const file of files) { + assert.ok(!readFileSync(file, 'utf8').includes(BANNED), `${file} references ${BANNED} - ${why}`); + } +} + +test('no CI workflow references the jev scripts', () => { + assertNoneReference(filesIn('.github/workflows', '.yml'), 'Jev must stay out of CI'); +}); + +test('no mise gate verb references the jev scripts', () => { + assertNoneReference(filesIn('mise', '.toml'), 'Jev must stay off the deterministic gate'); +}); diff --git a/scripts/jev/client.mjs b/scripts/jev/client.mjs new file mode 100644 index 0000000..adcb22c --- /dev/null +++ b/scripts/jev/client.mjs @@ -0,0 +1,81 @@ +// The Jev provider port for Foundry's advisory layer. Lifted from jev-triage's +// adapters, verified against https://docs.typesafe.ai/api (2026-09). +// +// ADVISORY ONLY. Nothing here may be imported by a gate verb (lint/typecheck/test/ +// audit) or by gate.yml / tier0.yml - ruleset_guard.py fails the build if it is. +// Jev routes where to spend expensive LLM effort; it never decides pass/fail. +// +// Zero dependencies; uses global fetch (Node 18+). + +const RETRYABLE = new Set([429, 529]); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** POST JSON with the docs' recommended exponential backoff on 429/529. */ +export async function postJson(url, headers, body, maxAttempts = 4) { + for (let attempt = 1; ; attempt++) { + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...headers }, + body: JSON.stringify(body), + }); + if (res.ok) return res.json(); + if (RETRYABLE.has(res.status) && attempt < maxAttempts) { + await sleep(250 * 2 ** (attempt - 1)); + continue; + } + throw new Error(`Jev request failed: ${res.status} ${await res.text()}`); + } +} + +/** TypeSafe first-party adapter: POST {baseUrl}/systemone, Bearer auth. */ +export class TypeSafeProvider { + constructor(apiKey, model = 'jev-latest', baseUrl = 'https://api.typesafe.ai/v1') { + this.apiKey = apiKey; + this.model = model; + this.baseUrl = baseUrl; + } + + async evaluate({ state, questions }) { + return postJson( + `${this.baseUrl}/systemone`, + { Authorization: `Bearer ${this.apiKey}` }, + { model: this.model, state, questions }, + ); + } +} + +/** Cloudflare Workers AI adapter: model in the body, no `result` envelope. */ +export class CloudflareProvider { + constructor(accountId, apiToken, model = 'typesafe/jev') { + this.accountId = accountId; + this.apiToken = apiToken; + this.model = model; + } + + async evaluate({ state, questions }) { + const url = `https://api.cloudflare.com/client/v4/accounts/${this.accountId}/ai/run`; + return postJson( + url, + { Authorization: `Bearer ${this.apiToken}` }, + { model: this.model, input: { state, questions } }, + ); + } +} + +/** + * Build a provider from the environment, or return null when no key is set so + * every caller fails open to current behavior (Jev is strictly opt-in). + * + * Reads: JEV_PROVIDER (typesafe|cloudflare), JEV_MODEL, JEV_API_KEY, + * CLOUDFLARE_ACCOUNT_ID, and TYPESAFE_AI_BASE_URL (self-host / proxy / mock). + */ +export function providerFromEnv(env = process.env) { + if ((env.JEV_PROVIDER ?? 'typesafe') === 'cloudflare') { + if (!env.CLOUDFLARE_ACCOUNT_ID || !env.JEV_API_KEY) return null; + return new CloudflareProvider(env.CLOUDFLARE_ACCOUNT_ID, env.JEV_API_KEY, env.JEV_MODEL || undefined); + } + if (!env.JEV_API_KEY) return null; + const baseUrl = env.TYPESAFE_AI_BASE_URL || 'https://api.typesafe.ai/v1'; + return new TypeSafeProvider(env.JEV_API_KEY, env.JEV_MODEL || 'jev-latest', baseUrl); +} diff --git a/scripts/jev/client.test.mjs b/scripts/jev/client.test.mjs new file mode 100644 index 0000000..783be12 --- /dev/null +++ b/scripts/jev/client.test.mjs @@ -0,0 +1,83 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { CloudflareProvider, TypeSafeProvider, postJson, providerFromEnv } from './client.mjs'; + +const realFetch = globalThis.fetch; + +function mockFetch(responses) { + const calls = []; + const queue = [...responses]; + globalThis.fetch = async (url, init) => { + calls.push({ url, init }); + const r = queue.shift(); + return { + ok: r.status >= 200 && r.status < 300, + status: r.status, + json: async () => r.body, + text: async () => JSON.stringify(r.body ?? ''), + }; + }; + return calls; +} + +async function withMock(responses, fn) { + const calls = mockFetch(responses); + try { + return await fn(calls); + } finally { + globalThis.fetch = realFetch; + } +} + +test('postJson posts JSON and parses the response', async () => { + await withMock([{ status: 200, body: { hello: 'world' } }], async (calls) => { + const out = await postJson('https://x/y', { Authorization: 'Bearer k' }, { a: 1 }); + assert.deepEqual(out, { hello: 'world' }); + assert.equal(calls[0].init.method, 'POST'); + assert.equal(calls[0].init.headers['Content-Type'], 'application/json'); + assert.equal(calls[0].init.headers.Authorization, 'Bearer k'); + assert.deepEqual(JSON.parse(calls[0].init.body), { a: 1 }); + }); +}); + +test('postJson retries on 429 then succeeds', async () => { + await withMock([{ status: 429 }, { status: 200, body: { ok: true } }], async (calls) => { + const out = await postJson('https://x/y', {}, {}); + assert.deepEqual(out, { ok: true }); + assert.equal(calls.length, 2); + }); +}); + +test('postJson throws on a non-retryable status', async () => { + await withMock([{ status: 500, body: 'boom' }], async () => { + await assert.rejects(() => postJson('https://x/y', {}, {}, 1), /500/); + }); +}); + +test('TypeSafeProvider posts to /systemone with model and auth', async () => { + await withMock([{ status: 200, body: { answers: {} } }], async (calls) => { + await new TypeSafeProvider('key', 'jev-latest').evaluate({ state: { q: 1 }, questions: { a: {} } }); + assert.match(calls[0].url, /\/v1\/systemone$/); + const body = JSON.parse(calls[0].init.body); + assert.equal(body.model, 'jev-latest'); + assert.deepEqual(body.state, { q: 1 }); + assert.equal(calls[0].init.headers.Authorization, 'Bearer key'); + }); +}); + +test('providerFromEnv returns null without a key (fail-open)', () => { + assert.equal(providerFromEnv({}), null); +}); + +test('providerFromEnv builds a TypeSafe provider and honors the base-url override', () => { + const p = providerFromEnv({ JEV_API_KEY: 'k', TYPESAFE_AI_BASE_URL: 'http://localhost:9/v1' }); + assert.ok(p instanceof TypeSafeProvider); + assert.equal(p.baseUrl, 'http://localhost:9/v1'); + assert.equal(p.model, 'jev-latest'); +}); + +test('providerFromEnv builds a Cloudflare provider, or null when the account is missing', () => { + const p = providerFromEnv({ JEV_PROVIDER: 'cloudflare', JEV_API_KEY: 'k', CLOUDFLARE_ACCOUNT_ID: 'acc' }); + assert.ok(p instanceof CloudflareProvider); + assert.equal(providerFromEnv({ JEV_PROVIDER: 'cloudflare', JEV_API_KEY: 'k' }), null); +}); diff --git a/scripts/jev/route.mjs b/scripts/jev/route.mjs new file mode 100644 index 0000000..4fbc88b --- /dev/null +++ b/scripts/jev/route.mjs @@ -0,0 +1,99 @@ +// Pure routing/triage core for Foundry's Jev advisory layer. No network, no I/O: +// it turns Jev answers plus config into decisions for the PROPOSER side (which +// diff hunks deserve deep review, which turns look risky). Unit-tested in isolation. +// +// Fail-open is the invariant: a missing or low-confidence answer always widens +// review, never narrows it. Jev can only ever ADD or DEEPEN review scope. + +/** Template defaults; overridden by the `[jev]` block in mise.toml. */ +export const DEFAULT_CONFIG = { + review: { depthThreshold: 1.5, minConfidence: 0.6, routeByLens: true }, + // holdAbove null => advisory only (warn, never hold) - the safe default. + toolcall: { warnAbove: 0.5, holdAbove: null }, +}; + +/** The batched review questions for one hunk (see docs/FEATURES). */ +export function reviewQuestions() { + return { + depth: { + type: 'score', + instructions: 'How much does this change warrant deep human-grade review?', + criteria: ['trivial', 'routine', 'worth-a-look', 'high-stakes'], + }, + lens: { + type: 'choice', + instructions: 'Which review lens is most relevant?', + criteria: { + correctness: 'logic or behavior bugs', + standards: 'repo conventions and style', + spec: 'matches the ticket intent', + }, + }, + touches_security: { + type: 'noul', + instructions: 'Does this hunk touch auth, secrets, or input handling?', + }, + }; +} + +/** The per-turn tool-call risk question. */ +export function toolcallQuestion() { + return { + type: 'noul', + instructions: + 'Did this turn include a destructive or irreversible action - deleting data, ' + + 'force-pushing, dropping a table, or calling a payment or side-effecting external API?', + }; +} + +/** Decide whether a hunk gets deep review and on which lens. */ +export function routeReview(answers, cfg = DEFAULT_CONFIG) { + const depthAnswer = answers?.depth; + const depth = scoreOf(depthAnswer); + const confidence = confidenceOf(depthAnswer); + const review = shouldReview(depth, confidence, cfg.review); + const lens = cfg.review.routeByLens ? choiceOf(answers?.lens) : null; + return { + review, + lens, + security: noulOf(answers?.touches_security), + reason: reviewReason(review, depth, confidence, cfg.review), + }; +} + +/** Triage one turn's tool-call risk into warn/hold decisions. */ +export function triageToolcall(answer, cfg = DEFAULT_CONFIG) { + const risk = noulOf(answer); + const tc = cfg.toolcall; + const warn = risk !== null && risk >= tc.warnAbove; + const hold = tc.holdAbove !== null && risk !== null && risk >= tc.holdAbove; + return { hold, warn, risk, reason: toolcallReason(risk, warn, hold) }; +} + +function shouldReview(depth, confidence, rc) { + if (depth === null) return true; // no answer -> review everything + if (confidence !== null && confidence < rc.minConfidence) return true; // unsure -> review + return depth >= rc.depthThreshold; +} + +function reviewReason(review, depth, confidence, rc) { + if (depth === null) return 'no depth score - baseline review'; + if (confidence !== null && confidence < rc.minConfidence) { + return `depth ${fmt(depth)} but low confidence ${fmt(confidence)} - review to be safe`; + } + const rel = review ? `>= ${rc.depthThreshold}` : `< ${rc.depthThreshold}`; + return review ? `depth ${fmt(depth)} ${rel} - deep review` : `depth ${fmt(depth)} ${rel} - skip (still gets baseline review)`; +} + +function toolcallReason(risk, warn, hold) { + if (risk === null) return 'no risk signal'; + if (hold) return `risk ${fmt(risk)} - hold for confirmation`; + if (warn) return `risk ${fmt(risk)} - advisory warning`; + return `risk ${fmt(risk)} - clear`; +} + +const scoreOf = (a) => (a && a.type === 'score' && typeof a.score === 'number' ? a.score : null); +const noulOf = (a) => (a && a.type === 'noul' && typeof a.noul === 'number' ? a.noul : null); +const choiceOf = (a) => (a && a.type === 'choice' && typeof a.choice === 'string' ? a.choice : null); +const confidenceOf = (a) => (a && typeof a.confidence === 'number' ? a.confidence : null); +const fmt = (n) => (n === null ? 'n/a' : Math.round(n * 100) / 100); diff --git a/scripts/jev/route.test.mjs b/scripts/jev/route.test.mjs new file mode 100644 index 0000000..3b23dfe --- /dev/null +++ b/scripts/jev/route.test.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { DEFAULT_CONFIG, reviewQuestions, routeReview, toolcallQuestion, triageToolcall } from './route.mjs'; + +const depth = (score, confidence) => ({ type: 'score', score, confidence }); +const lens = (choice, confidence = 0.8) => ({ type: 'choice', choice, confidence, probabilities: {} }); +const noul = (n) => ({ type: 'noul', noul: n }); + +test('high-depth hunk gets deep review and its lens', () => { + const out = routeReview({ depth: depth(2.7, 0.86), lens: lens('correctness'), touches_security: noul(0.12) }); + assert.equal(out.review, true); + assert.equal(out.lens, 'correctness'); + assert.equal(out.security, 0.12); +}); + +test('low-depth, confident hunk is skipped', () => { + const out = routeReview({ depth: depth(1.0, 0.9) }); + assert.equal(out.review, false); + assert.match(out.reason, /skip/); +}); + +test('low confidence forces review even when depth is below threshold', () => { + const out = routeReview({ depth: depth(0.5, 0.4) }); + assert.equal(out.review, true); + assert.match(out.reason, /confidence/); +}); + +test('a missing depth answer fails open to review', () => { + const out = routeReview({}); + assert.equal(out.review, true); +}); + +test('lens routing can be disabled', () => { + const cfg = { ...DEFAULT_CONFIG, review: { ...DEFAULT_CONFIG.review, routeByLens: false } }; + const out = routeReview({ depth: depth(3, 0.9), lens: lens('spec') }, cfg); + assert.equal(out.lens, null); +}); + +test('tool-call risk warns but never holds by default', () => { + const out = triageToolcall(noul(0.8)); + assert.equal(out.warn, true); + assert.equal(out.hold, false); +}); + +test('hold fires only when holdAbove is configured', () => { + const cfg = { ...DEFAULT_CONFIG, toolcall: { warnAbove: 0.5, holdAbove: 0.9 } }; + assert.equal(triageToolcall(noul(0.95), cfg).hold, true); + assert.equal(triageToolcall(noul(0.6), cfg).hold, false); +}); + +test('a missing risk answer neither warns nor holds', () => { + const out = triageToolcall(undefined); + assert.equal(out.warn, false); + assert.equal(out.hold, false); + assert.equal(out.risk, null); +}); + +test('question templates carry the right primitive types', () => { + const q = reviewQuestions(); + assert.equal(q.depth.type, 'score'); + assert.equal(q.lens.type, 'choice'); + assert.equal(q.touches_security.type, 'noul'); + assert.equal(toolcallQuestion().type, 'noul'); +}); From b5e668693d4858ee0deae5c06736ef818afadd58 Mon Sep 17 00:00:00 2001 From: CMaintz Date: Wed, 30 Sep 2026 18:58:09 +0200 Subject: [PATCH 2/2] feat(jev): finish v0.1 - runnable review pre-filter + CI + precise guard Completes the advisory Jev layer so v0.1 is actually usable, not just a library. - scripts/jev/review.mjs: runnable review pre-filter. `node scripts/jev/review.mjs [baseRef]` parses the diff, asks Jev per changed file which warrant deep review and on which lens, and prints routing JSON. Fail-open: no key or a small diff reviews everything. Exported routeFiles/parseDiff are unit-tested with a fake provider. - jev-scripts.yml: runs `node --test scripts/jev/*.test.mjs` on change, so the suite - including the boundary guard - actually executes in CI and has teeth. - boundary.test.mjs now scans the gate-defining files explicitly (gate/security/_ts/ _java/_dotnet/_php/_guards/_semgrep + mise verbs) rather than every workflow, so the test-runner workflow above is allowed while Jev-in-the-gate is still a red build. - FEATURES + README updated. 22 node:test cases, all green. --- .github/workflows/jev-scripts.yml | 21 +++++++++ docs/FEATURES.md | 2 +- scripts/jev/README.md | 10 +++++ scripts/jev/boundary.test.mjs | 39 +++++++++++------ scripts/jev/review.mjs | 71 +++++++++++++++++++++++++++++++ scripts/jev/review.test.mjs | 57 +++++++++++++++++++++++++ 6 files changed, 187 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/jev-scripts.yml create mode 100644 scripts/jev/review.mjs create mode 100644 scripts/jev/review.test.mjs diff --git a/.github/workflows/jev-scripts.yml b/.github/workflows/jev-scripts.yml new file mode 100644 index 0000000..a914fdb --- /dev/null +++ b/.github/workflows/jev-scripts.yml @@ -0,0 +1,21 @@ +name: jev-scripts +# Runs the advisory Jev layer's own unit tests (node:test, zero deps), including the +# boundary guard that keeps Jev out of the deterministic gate. NOT a gate itself - it +# tests the proposer-side scripts. Only fires when those scripts change. +on: + pull_request: + paths: ['scripts/jev/**'] + push: + branches: [main] + paths: ['scripts/jev/**'] +permissions: + contents: read +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + - run: node --test scripts/jev/*.test.mjs diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 36ab12f..4bf0c17 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -75,7 +75,7 @@ fails the verb instead of going unchecked. - `npm-audit-ratchet.mjs` — ratcheted `npm audit` for the ts `audit` verb: fails on any critical not in `.audit-allowlist.json` and on stale entries, so accepted CVE debt can only shrink (npm audit has no native per-advisory ignore). Degrades to plain `npm audit --audit-level=critical` with no allowlist. Reads the report from stdin. - `foundry-init.sh` — one-shot repo scaffold. - `setup-labels.sh` — create the GitHub labels the workflows + ticket state machine need (agent:ready/working/blocked, align, ruleset-change, autofix). Idempotent; run by `foundry-init`. -- `jev/` — the **advisory** Jev layer (`client.mjs` provider port lifted from jev-triage; `route.mjs` pure routing/triage core). Near-free System-One decisions on the **proposer** side only: which diff hunks warrant deep review and on which lens, and whether a turn looks destructive. Opt-in via `JEV_API_KEY` (+ `JEV_PROVIDER`/`JEV_MODEL`/`TYPESAFE_AI_BASE_URL`); absent key ⇒ every caller fails open to current behavior. **Never in the deterministic gate** — `boundary.test.mjs` reds the build if any workflow or mise verb references it. Tests: `node --test scripts/jev/*.test.mjs`. +- `jev/` — the **advisory** Jev layer: `client.mjs` (provider port lifted from jev-triage), `route.mjs` (pure routing/triage core), and `review.mjs` (runnable review pre-filter: `node scripts/jev/review.mjs [baseRef]` prints per-file `{review, lens, reason}` routing JSON). Near-free System-One decisions on the **proposer** side only: which diff hunks warrant deep review and on which lens. Opt-in via `JEV_API_KEY` (+ `JEV_PROVIDER`/`JEV_MODEL`/`TYPESAFE_AI_BASE_URL`); absent key ⇒ every caller fails open to current behavior (review all). **Never in the deterministic gate** — `boundary.test.mjs` reds the build if a gate workflow or mise verb references it; the `jev-scripts.yml` workflow runs the suite (`node --test scripts/jev/*.test.mjs`) on change. ## Agent half — `cmaintz-skills` diff --git a/scripts/jev/README.md b/scripts/jev/README.md index 62af165..0e956bf 100644 --- a/scripts/jev/README.md +++ b/scripts/jev/README.md @@ -38,6 +38,16 @@ const { review, lens, reason } = routeReview(answers); // review only the above-threshold hunks, on `lens`; log the rest (still get baseline review). ``` +Or run the ready-made review pre-filter over a diff and consume its routing JSON: + +``` +node scripts/jev/review.mjs [baseRef] # baseRef defaults to origin/main +JEV_REVIEW_MIN_FILES=8 node scripts/jev/review.mjs # skip Jev on small diffs +``` + +It prints, per changed file, `{ review, lens, reason }`. No key (or a diff at/under the +min-files gate) routes every file to review - Jev only ever narrows spend, never the net. + ## Config Read from the environment (keys never in CI): diff --git a/scripts/jev/boundary.test.mjs b/scripts/jev/boundary.test.mjs index 45636ac..b3af416 100644 --- a/scripts/jev/boundary.test.mjs +++ b/scripts/jev/boundary.test.mjs @@ -1,20 +1,34 @@ import assert from 'node:assert/strict'; -import { readdirSync, readFileSync } from 'node:fs'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { test } from 'node:test'; // The invariant (spec foundry-jev-integration, deterministic oracle / probabilistic -// proposer): Jev is advisory-only and must NEVER be referenced by a gate-defining -// file - not a CI workflow, not a mise gate verb. A reference here would mean a -// ~68%-accurate model crept into the authoritative pass/fail path. This test makes -// that a red build rather than a code-review hope. Run from the repo root. +// proposer): Jev is advisory-only and must NEVER be referenced by a GATE-defining file - +// a gate workflow or a mise verb. A reference there would put a ~68%-accurate model in +// the authoritative pass/fail path. This test reds the build if that happens. +// +// It scans the gate files explicitly (not every workflow) so a separate test-runner +// workflow that runs these very tests is allowed - that is not the gate. Run from the repo root. const BANNED = 'scripts/jev'; -function filesIn(dir, ext) { +const GATE_WORKFLOWS = [ + 'gate.yml', + 'security.yml', + 'tier0.yml', + '_ts.yml', + '_java.yml', + '_dotnet.yml', + '_php.yml', + '_guards.yml', + '_semgrep.yml', +].map((name) => join('.github/workflows', name)); + +function miseFiles() { try { - return readdirSync(dir) - .filter((name) => name.endsWith(ext)) - .map((name) => join(dir, name)); + return readdirSync('mise') + .filter((name) => name.endsWith('.toml')) + .map((name) => join('mise', name)); } catch { return []; } @@ -22,14 +36,15 @@ function filesIn(dir, ext) { function assertNoneReference(files, why) { for (const file of files) { + if (!existsSync(file)) continue; assert.ok(!readFileSync(file, 'utf8').includes(BANNED), `${file} references ${BANNED} - ${why}`); } } -test('no CI workflow references the jev scripts', () => { - assertNoneReference(filesIn('.github/workflows', '.yml'), 'Jev must stay out of CI'); +test('no gate workflow references the jev scripts', () => { + assertNoneReference(GATE_WORKFLOWS, 'Jev must stay out of the CI gate'); }); test('no mise gate verb references the jev scripts', () => { - assertNoneReference(filesIn('mise', '.toml'), 'Jev must stay off the deterministic gate'); + assertNoneReference(miseFiles(), 'Jev must stay off the deterministic gate'); }); diff --git a/scripts/jev/review.mjs b/scripts/jev/review.mjs new file mode 100644 index 0000000..7de43f1 --- /dev/null +++ b/scripts/jev/review.mjs @@ -0,0 +1,71 @@ +#!/usr/bin/env node +// v0.1 review pre-filter (runnable). Asks Jev, per changed file, whether it warrants +// deep review and on which lens, then prints routing JSON the review skill consumes. +// +// Advisory and fail-open: no key, or a diff at/under the min-files gate, routes every +// file to review (current behavior) - Jev only ever narrows spend, never the safety net. +// +// node scripts/jev/review.mjs [baseRef] # baseRef defaults to origin/main +// JEV_REVIEW_MIN_FILES=8 node scripts/jev/review.mjs # skip Jev on small diffs + +import { execSync } from 'node:child_process'; +import { providerFromEnv } from './client.mjs'; +import { reviewQuestions, routeReview } from './route.mjs'; + +/** Route each changed file: ask Jev per file, or review-all when Jev is unavailable. */ +export async function routeFiles(provider, files, minFiles = 0) { + if (!provider || files.length <= minFiles) { + const reason = provider ? `at or below the ${minFiles}-file gate` : 'no JEV_API_KEY (review all)'; + return files.map((f) => ({ file: f.file, review: true, lens: null, reason })); + } + const routed = []; + for (const { file, patch } of files) { + const { answers } = await provider.evaluate({ state: { file, diff: patch }, questions: reviewQuestions() }); + routed.push({ file, ...routeReview(answers) }); + } + return routed; +} + +/** Split `git diff` text into one patch per file (skips deletions to /dev/null). */ +export function parseDiff(text) { + const out = []; + let current = null; + for (const line of text.split('\n')) { + if (line.startsWith('diff --git')) { + if (current) out.push({ file: current.file, patch: current.lines.join('\n') }); + current = { file: fileFromHeader(line), lines: [line] }; + } else if (current) { + if (line.startsWith('+++ ')) current.file = plusPath(line) ?? current.file; + current.lines.push(line); + } + } + if (current) out.push({ file: current.file, patch: current.lines.join('\n') }); + return out.filter((d) => d.file !== '/dev/null'); +} + +function fileFromHeader(line) { + const match = /^diff --git a\/.+ b\/(.+)$/.exec(line); + return match?.[1] ?? 'unknown'; +} + +function plusPath(line) { + const path = line.slice(4).trim(); + if (path === '/dev/null') return '/dev/null'; + return path.startsWith('b/') ? path.slice(2) : path; +} + +async function main() { + const baseRef = process.argv[2] ?? 'origin/main'; + const diff = execSync(`git diff ${baseRef}`, { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }); + const minFiles = Number(process.env.JEV_REVIEW_MIN_FILES ?? '0'); + const routed = await routeFiles(providerFromEnv(), parseDiff(diff), minFiles); + process.stdout.write(`${JSON.stringify(routed, null, 2)}\n`); +} + +// Run only as a CLI, not when imported by tests. +if (import.meta.url === `file://${process.argv[1]}` || process.argv[1]?.endsWith('review.mjs')) { + main().catch((err) => { + process.stderr.write(`jev-review: ${err?.message ?? err}\n`); + process.exit(0); // advisory: never break the caller + }); +} diff --git a/scripts/jev/review.test.mjs b/scripts/jev/review.test.mjs new file mode 100644 index 0000000..946ce72 --- /dev/null +++ b/scripts/jev/review.test.mjs @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { parseDiff, routeFiles } from './review.mjs'; + +const DIFF = [ + 'diff --git a/src/core/decide.ts b/src/core/decide.ts', + 'index 111..222 100644', + '--- a/src/core/decide.ts', + '+++ b/src/core/decide.ts', + '@@ -1 +1 @@', + '-old', + '+new', + 'diff --git a/README.md b/README.md', + '--- a/README.md', + '+++ b/README.md', + '@@ -1 +1 @@', + '+docs', +].join('\n'); + +test('parseDiff splits a diff into one patch per file', () => { + const files = parseDiff(DIFF); + assert.equal(files.length, 2); + assert.equal(files[0].file, 'src/core/decide.ts'); + assert.equal(files[1].file, 'README.md'); + assert.match(files[0].patch, /\+new/); +}); + +test('routeFiles reviews everything when there is no provider (fail open)', async () => { + const routed = await routeFiles(null, parseDiff(DIFF)); + assert.equal(routed.length, 2); + assert.ok(routed.every((r) => r.review === true)); + assert.match(routed[0].reason, /no JEV_API_KEY/); +}); + +test('routeFiles skips Jev entirely under the min-files gate', async () => { + let calls = 0; + const provider = { evaluate: async () => (calls++, { answers: {} }) }; + const routed = await routeFiles(provider, parseDiff(DIFF), 5); + assert.equal(calls, 0); + assert.ok(routed.every((r) => r.review === true)); +}); + +test('routeFiles asks Jev per file and routes on the answers', async () => { + const provider = { + evaluate: async ({ state }) => ({ + answers: + state.file === 'README.md' + ? { depth: { type: 'score', score: 0.2, confidence: 0.9 } } + : { depth: { type: 'score', score: 3, confidence: 0.9 }, lens: { type: 'choice', choice: 'correctness', confidence: 0.8 } }, + }), + }; + const routed = await routeFiles(provider, parseDiff(DIFF), 0); + const byFile = Object.fromEntries(routed.map((r) => [r.file, r])); + assert.equal(byFile['src/core/decide.ts'].review, true); + assert.equal(byFile['src/core/decide.ts'].lens, 'correctness'); + assert.equal(byFile['README.md'].review, false); +});