diff --git a/changelog.d/425-evolution-dataset.added.md b/changelog.d/425-evolution-dataset.added.md new file mode 100644 index 00000000..db0aa2c7 --- /dev/null +++ b/changelog.d/425-evolution-dataset.added.md @@ -0,0 +1 @@ +`npm run evolution:export` now writes one dataset with a stable identity instead of a bare row array: a `ds_` id, the source window, the repositories, the row count, a sha256 `contentHash` over the canonical manifest and rows, a train/holdout split that keeps every repository on one side, and the holdout entries a promotion must not repeat. Every exported string is passed through the platform redactor, and only repositories an admin opted in with `npm run evolution:consent -- --repo owner/name --on` appear at all; consent is off by default and every change is audited. `--report` prints the manifest's own split counts, and the exporter still opens the database read-only and never writes to it. diff --git a/docs/architecture.md b/docs/architecture.md index e52ae317..b3878fb1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -925,9 +925,38 @@ held-out evidence, and a human-approved decision. AgentDescent is an offline engine behind this ledger; it never runs a user repository, never receives a GitHub token and cannot change the evaluator, CI or merge gate. See [decisions/0003](decisions/0003-agentdescent-evolution.md). The read-only -`scripts/evolution-shadow.mjs` exporter turns recent jobs into a redacted -dataset, and `--report` prints the baseline plus a time-ordered train/holdout -split; it does not write to the database. +`scripts/evolution-shadow.ts` exporter turns recent jobs into one dataset with a +manifest. The manifest records the `ds_` id, the selection (`days` and the +applied `limit`), the repositories, `rowCount`, the effective +`trainRows`/`holdoutRows`, the `split.train`/`split.holdout` job-id arrays, the +holdout's protected entries, and a `window` that is provenance only: nominal +`from`/`to` bounds plus `oldestRowAt`/`newestRowAt`, the real coverage of the +rows that were written. The bounds are anchored on `newestRowAt` unless the +caller passes an explicit `now`, while the exporter's SQL bound is the +wall-clock `now - days`; they are the requested extent, not a claim that the +rows reach it. `--limit` truncates to the newest N inside `--days`, so +`oldestRowAt`/`newestRowAt`, not the window bound, say how far the export +actually reaches. + +`contentHash` is sha256 over the selection, the split and the canonical, redacted +rows; the window, the id and the hash itself are outside the hashed object. `id` +is `ds_` plus the first 16 hex characters of that hash, so the same rows and +parameters produce the same hash twice. `--verify ` recomputes the split +from the records, the hash and the id, and refuses with `SPLIT_MISMATCH`, +`MANIFEST_MISMATCH`, `CONTENT_HASH_MISMATCH`, `DATASET_ID_MISMATCH` or +`MALFORMED_DATASET`; a moved job id fails even if someone recomputes the hash. + +The split is repository-disjoint first: the newest 20% of rows by time only +chooses which repositories are held out, and every row of a repository named in +that tail moves to the holdout — so the holdout share can exceed 20%, and a +holdout row can be older than a train row. `#426`–`#432` read the split from the +manifest rather than assuming its size. The exporter refuses with +`NO_TRAINING_ROWS` and writes nothing when the split would hold out every row, +for example with a single consented repository. Every exported string passes +through `redact()`, and the exporter reads only repositories that opted in +through the audited `evolution.consent` command; consent is off by default. +`--report` prints the baseline and the manifest's own split counts; the export +does not write to the database. The ledger's shape is the table that already existed, extended rather than replaced: `artifact_versions` gains `kind`, `scope`, `repo_id`, `parent_id` (the diff --git a/docs/decisions/0003-agentdescent-evolution.md b/docs/decisions/0003-agentdescent-evolution.md index f71d90d1..7d4c0a27 100644 --- a/docs/decisions/0003-agentdescent-evolution.md +++ b/docs/decisions/0003-agentdescent-evolution.md @@ -123,10 +123,17 @@ frozen path is refused before evaluation, with `evolution.candidate.refused` in the audit log. `job_events` stores the durable timeline that makes a job replayable. The -`scripts/evolution-shadow.mjs` export is read-only and excludes tokens, -callback secrets and API keys. Run `npm run evolution:report` (or pass `--report`) to print the current -baseline (success/merge rate, evidence completeness, median duration) and a -time-ordered train/holdout split before any candidate is generated. +`scripts/evolution-shadow.ts` export is read-only and excludes tokens, +callback secrets and API keys: every exported string passes through `redact()`, +and only repositories an admin opted in through the audited consent command are +read. Run `npm run evolution:report` (or pass `--report`) to print the current +baseline (success/merge rate, evidence completeness, median duration) and the +manifest's own train/holdout split before any candidate is generated. The split +is repository-disjoint first: the newest time tail only chooses which +repositories are held out, a repository spanning the cut moves all of its rows, +and `trainRows`/`holdoutRows` record the effective sizes. An export whose +holdout would take every row is refused with `NO_TRAINING_ROWS`, and +`--verify` recomputes a written dataset's hash and id. ## Security and privacy diff --git a/docs/security.md b/docs/security.md index 8cc68c01..6b809f16 100644 --- a/docs/security.md +++ b/docs/security.md @@ -667,6 +667,11 @@ Upstream `api_key:` hashes from log messages. Request bodies are never logged. The development mail outbox is written with mode 0600 and contains approval links; treat it as a secret. +The evolution export uses a read-only database handle and passes every exported +string through the same `redact()` before serialisation. It reads only +repositories an admin opted in through the audited consent command, so a +repository that never consented is absent from the dataset and its holdout. + ### Headers - `Content-Security-Policy` for HTML pages: self only, no inline scripts, images diff --git a/package.json b/package.json index ee384bf5..1d12cd99 100644 --- a/package.json +++ b/package.json @@ -23,14 +23,15 @@ "db:restore-drill": "node scripts/restore-drill.mjs", "role:set": "node scripts/set-role.mjs", "role:set:dev": "tsx scripts/set-role.ts", - "evolution:export": "node scripts/evolution-shadow.mjs", - "evolution:report": "node scripts/evolution-shadow.mjs --report", + "evolution:export": "tsx scripts/evolution-shadow.ts", + "evolution:report": "tsx scripts/evolution-shadow.ts --report", "db:restore-drill:ci": "node scripts/restore-drill-ci.mjs", "artifacts:prune": "node scripts/artifact-retention.mjs", "secret:scan": "node scripts/secret-scan.mjs", "slo:report": "node scripts/slo-report.mjs", "trial:sweep": "tsx scripts/trial-sweep.ts", "visual:matrix": "tsx scripts/visual-matrix.ts", + "evolution:consent": "tsx scripts/evolution-consent.ts", "lint:update": "node scripts/lint-warnings.mjs --update" }, "dependencies": { diff --git a/scripts/evolution-consent.ts b/scripts/evolution-consent.ts new file mode 100644 index 00000000..7a90f622 --- /dev/null +++ b/scripts/evolution-consent.ts @@ -0,0 +1,45 @@ +#!/usr/bin/env node +// #425: the consent switch. A repository enters an evolution dataset only when an +// admin or superadmin turns it on here; the change is audited and the exporter +// reads nothing else. `#235` owns the settings-card control, so this slice ships +// the operator path only. +// +// Run with tsx: `npm run evolution:consent -- --repo owner/name --on --actor ` +import { loadConfig } from '../src/config.js' +import { openStore } from '../src/db.js' + +function arg(name: string, fallback = ''): string { + const index = process.argv.indexOf(name) + return index === -1 ? fallback : (process.argv[index + 1] ?? fallback) +} + +const repoName = arg('--repo', '').trim() +const on = process.argv.includes('--on') +const off = process.argv.includes('--off') +const actorLogin = arg('--actor', process.env.EVOLUTION_ACTOR ?? '').trim() + +if (repoName === '' || on === off || actorLogin === '') { + console.error('usage: npm run evolution:consent -- --repo <--on|--off> --actor ') + process.exit(1) +} + +const config = loadConfig() +const store = openStore(config.dbPath) +try { + const actor = store.userByLogin(actorLogin) + if (actor === undefined) throw new Error(`actor not found: ${actorLogin}`) + // The operator path is audited; the role check is what keeps a non-admin from + // widening the dataset, exactly as the console would. + if (actor.role !== 'admin' && actor.role !== 'superadmin') { + throw new Error(`refused: ${actorLogin} is ${actor.role}, and consent is admin-only`) + } + const target = store.listEvolutionConsent().find((row) => row.repo.toLowerCase() === repoName.toLowerCase()) + if (target === undefined) throw new Error(`repository not found: ${repoName}`) + store.setRepoEvolutionConsent(target.repoId, on, actor.id) + console.log(`consent ${on ? 'on' : 'off'}: ${target.repo}`) +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 +} finally { + store.close() +} diff --git a/scripts/evolution-shadow.mjs b/scripts/evolution-shadow.mjs deleted file mode 100755 index dbf3bcc6..00000000 --- a/scripts/evolution-shadow.mjs +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env node -import { existsSync, writeFileSync } from 'node:fs' -import { createRequire } from 'node:module' -import { resolve } from 'node:path' - -const require = createRequire(import.meta.url) -const { DatabaseSync } = require('node:sqlite') - -function arg(name, fallback = '') { - const index = process.argv.indexOf(name) - return index === -1 ? fallback : (process.argv[index + 1] ?? fallback) -} - -const dbPath = resolve(arg('--db', process.env.DB_PATH ?? 'data/fledge.db')) -const days = Math.max(1, Math.min(365, Number(arg('--days', '90')) || 90)) -const limit = Math.max(1, Math.min(5000, Number(arg('--limit', '500')) || 500)) -const repoFilter = arg('--repo', '') -const out = arg('--out', '') -const summaryOnly = process.argv.includes('--summary') -const reportOnly = process.argv.includes('--report') - -if (!existsSync(dbPath)) { - console.error(`database not found: ${dbPath}`) - process.exit(1) -} - -const db = new DatabaseSync(dbPath, { readOnly: true }) -const version = Number(db.prepare('PRAGMA user_version').get()?.user_version ?? 0) -if (version > 28) { - console.error(`database schema ${version} is newer than this exporter supports (28)`) - process.exit(1) -} -const jobColumns = new Set(db.prepare('PRAGMA table_info(jobs)').all().map((column) => column.name)) -if (!jobColumns.has('kind') || !jobColumns.has('evidence_json') || !jobColumns.has('team_stats_json')) { - console.error('database schema is older than the current job evidence columns; start the current Fledge once before exporting') - process.exit(1) -} - -const since = new Date(Date.now() - days * 86_400_000).toISOString() -const rows = db.prepare(` - SELECT j.id, j.kind, j.status, j.trigger, j.goal, j.branch, j.pr_url, j.issue_url, - j.review_score, j.evidence_json, j.attribution_json, j.team_stats_json, - j.checkpoint, j.base_sha, j.cost_usd, j.created_at, j.started_at, j.finished_at, - r.owner, r.name, r.upstream_owner, r.upstream_name, r.goal AS repo_goal, - r.stage_mode, r.repo_role, r.auto_merge - FROM jobs j - JOIN repos r ON r.id = j.repo_id - WHERE j.created_at >= ? - ORDER BY j.created_at DESC - LIMIT ? -`).all(since, limit) - -function parse(value) { - if (typeof value !== 'string' || value === '') return null - try { return JSON.parse(value) } catch { return null } -} - -const records = rows - .map((row) => ({ - jobId: row.id, - repo: `${row.upstream_owner || row.owner}/${row.upstream_name || row.name}`, - repoGoal: row.repo_goal ?? '', - kind: row.kind, - status: row.status, - trigger: row.trigger, - goal: row.goal, - baseSha: row.base_sha, - branch: row.branch, - prUrl: row.pr_url, - issueUrl: row.issue_url, - reviewScore: row.review_score, - evidence: parse(row.evidence_json), - attribution: parse(row.attribution_json), - teamStats: parse(row.team_stats_json), - checkpoint: row.checkpoint, - costUsd: Number(row.cost_usd ?? 0), - createdAt: row.created_at, - startedAt: row.started_at, - finishedAt: row.finished_at, - stageMode: row.stage_mode, - repoRole: row.repo_role, - autoMerge: row.auto_merge === 1, - })) - .filter((row) => repoFilter === '' || row.repo === repoFilter) - -function median(values) { - if (values.length === 0) return null - const sorted = [...values].sort((a, b) => a - b) - const middle = Math.floor(sorted.length / 2) - return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle] -} - -function duration(start, end) { - if (typeof start !== 'string' || typeof end !== 'string') return null - const a = Date.parse(start) - const b = Date.parse(end) - return Number.isFinite(a) && Number.isFinite(b) && b >= a ? b - a : null -} - -if (reportOnly) { - const succeeded = records.filter((row) => row.status === 'succeeded') - const failed = records.filter((row) => row.status === 'failed') - const prs = records.filter((row) => row.prUrl !== null) - const merged = records.filter((row) => row.status === 'succeeded' && row.prUrl !== null) - const scores = records.flatMap((row) => (row.kind === 'review' && typeof row.reviewScore === 'number' ? [row.reviewScore] : [])) - const evidenceComplete = records.filter((row) => { - const evidence = row.evidence ?? {} - if (row.kind === 'verify') return Boolean(evidence.testRed && evidence.testGreen) - if (row.kind === 'iterate') return Boolean(evidence.scope && evidence.acceptance && evidence.testRed && evidence.testGreen) - if (row.kind === 'review') return Boolean(row.reviewScore !== null) - return Boolean(evidence.scope || evidence.testPlan || evidence.testGreen) - }) - const durations = records.flatMap((row) => { - const value = duration(row.startedAt ?? row.createdAt, row.finishedAt) - return value === null ? [] : [value] - }) - const sorted = [...records].sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt))) - const holdoutCount = Math.max(1, Math.floor(sorted.length * 0.2)) - const train = sorted.slice(0, Math.max(0, sorted.length - holdoutCount)) - const holdout = sorted.slice(Math.max(0, sorted.length - holdoutCount)) - const byKind = {} - for (const row of records) byKind[row.kind] = (byKind[row.kind] ?? 0) + 1 - const byRepo = {} - for (const row of records) byRepo[row.repo] = (byRepo[row.repo] ?? 0) + 1 - console.log(JSON.stringify({ - database: dbPath, - schemaVersion: version, - windowDays: days, - records: records.length, - baseline: { - succeeded: succeeded.length, - failed: failed.length, - successRate: records.length === 0 ? null : Math.round((succeeded.length / records.length) * 1000) / 10, - prs: prs.length, - merged: merged.length, - mergeRate: prs.length === 0 ? null : Math.round((merged.length / prs.length) * 1000) / 10, - reviewAverage: scores.length === 0 ? null : Math.round((scores.reduce((sum, score) => sum + score, 0) / scores.length) * 10) / 10, - reviewCount: scores.length, - evidenceCompleteness: records.length === 0 ? null : Math.round((evidenceComplete.length / records.length) * 1000) / 10, - medianJobMs: median(durations), - }, - split: { train: train.length, holdout: holdout.length, holdoutFrom: holdout[0]?.createdAt ?? null }, - byKind, - byRepo, - }, null, 2)) - db.close() - process.exit(0) -} - -if (summaryOnly) { - const byStatus = new Map() - const byKind = new Map() - for (const row of records) { - byStatus.set(row.status, (byStatus.get(row.status) ?? 0) + 1) - byKind.set(row.kind, (byKind.get(row.kind) ?? 0) + 1) - } - console.log(JSON.stringify({ - database: dbPath, - schemaVersion: version, - days, - records: records.length, - byStatus: Object.fromEntries(byStatus), - byKind: Object.fromEntries(byKind), - }, null, 2)) - db.close() - process.exit(0) -} - -const payload = { - schemaVersion: version, - exportedAt: new Date().toISOString(), - days, - count: records.length, - records, -} -const text = `${JSON.stringify(payload, null, 2)}\n` -if (out !== '') { - writeFileSync(resolve(out), text) - console.error(`wrote ${records.length} records to ${resolve(out)}`) -} else { - process.stdout.write(text) -} -db.close() diff --git a/scripts/evolution-shadow.ts b/scripts/evolution-shadow.ts new file mode 100644 index 00000000..a0e82f53 --- /dev/null +++ b/scripts/evolution-shadow.ts @@ -0,0 +1,176 @@ +#!/usr/bin/env node +// #425: the read-only evolution export. Same CLI as the `.mjs` it replaces +// (`--db`, `--days`, `--limit`, `--repo`, `--out`, `--report`, `--summary`, +// `--verify`) but the payload carries a manifest with its canonical hash, every +// string has passed `redact()`, and only repositories that consented are read. +// The open stays read-only and this script never writes to the database. +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { DatabaseSync } from 'node:sqlite' +import { assertExportable, buildDataset, verifyDataset } from '../src/lib/evolution/dataset.js' +import { loadEvolutionRows } from '../src/lib/evolution/export.js' + +function arg(name: string, fallback = ''): string { + const index = process.argv.indexOf(name) + return index === -1 ? fallback : (process.argv[index + 1] ?? fallback) +} + +const dbPath = resolve(arg('--db', process.env.DB_PATH ?? 'data/fledge.db')) +const days = Math.max(1, Math.min(365, Number(arg('--days', '90')) || 90)) +const limit = Math.max(1, Math.min(5000, Number(arg('--limit', '500')) || 500)) +const repoFilter = arg('--repo', '') +const out = arg('--out', '') +const verifyPath = arg('--verify', '') +const summaryOnly = process.argv.includes('--summary') +const reportOnly = process.argv.includes('--report') + +// `--verify` reads a written dataset and recomputes it. It needs no database, so +// it runs before the database is opened; a mismatch is a named refusal and exit 1. +if (verifyPath !== '') { + let payload: unknown + try { + payload = JSON.parse(readFileSync(resolve(verifyPath), 'utf8')) + } catch (error) { + console.error(`verify: MALFORMED_DATASET ${error instanceof Error ? error.message : String(error)}`) + process.exit(1) + } + const verdict = verifyDataset(payload) + if (!verdict.ok) { + console.error(`verify: ${verdict.code} ${verdict.detail}`) + process.exit(1) + } + console.log(`verify: ok ${verdict.manifest.id} ${verdict.manifest.contentHash}`) + process.exit(0) +} + +if (!existsSync(dbPath)) { + console.error(`database not found: ${dbPath}`) + process.exit(1) +} + +const db = new DatabaseSync(dbPath, { readOnly: true }) +const version = Number(db.prepare('PRAGMA user_version').get()?.user_version ?? 0) +const jobColumns = new Set(db.prepare('PRAGMA table_info(jobs)').all().map((column) => String(column.name))) +const repoColumns = new Set(db.prepare('PRAGMA table_info(repos)').all().map((column) => String(column.name))) +try { + assertExportable(jobColumns, version) + if (!repoColumns.has('evolution_consent')) throw new Error('database has no repos.evolution_consent column; start the current Fledge once before exporting') +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exit(1) +} + +const since = new Date(Date.now() - days * 86_400_000).toISOString() +const records = loadEvolutionRows(db, { since, limit }).filter((row) => repoFilter === '' || row.repo === repoFilter) + +const dataset = buildDataset(records, { days, limit }) +try { + // The split refuses to export a dataset with no training side: a named refusal + // is better than a file nobody can train on. + assertExportable(jobColumns, version, dataset.manifest.trainRows) +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + db.close() + process.exit(1) +} + +function median(values: number[]): number | null { + if (values.length === 0) return null + const sorted = [...values].sort((a, b) => a - b) + const middle = Math.floor(sorted.length / 2) + return sorted.length % 2 === 0 ? (sorted[middle - 1]! + sorted[middle]!) / 2 : sorted[middle]! +} + +function duration(start: string | null, end: string | null): number | null { + if (typeof start !== 'string' || typeof end !== 'string') return null + const a = Date.parse(start) + const b = Date.parse(end) + return Number.isFinite(a) && Number.isFinite(b) && b >= a ? b - a : null +} + +if (reportOnly) { + const exported = dataset.records + const succeeded = exported.filter((row) => row.status === 'succeeded') + const failed = exported.filter((row) => row.status === 'failed') + const prs = exported.filter((row) => row.prUrl !== null) + const merged = exported.filter((row) => row.status === 'succeeded' && row.prUrl !== null) + const scores = exported.flatMap((row) => (row.kind === 'review' && typeof row.reviewScore === 'number' ? [row.reviewScore] : [])) + const evidenceComplete = exported.filter((row) => { + const evidence = (row.evidence ?? {}) as Record + if (row.kind === 'verify') return Boolean(evidence.testRed && evidence.testGreen) + if (row.kind === 'iterate') return Boolean(evidence.scope && evidence.acceptance && evidence.testRed && evidence.testGreen) + if (row.kind === 'review') return Boolean(row.reviewScore !== null) + return Boolean(evidence.scope || evidence.testPlan || evidence.testGreen) + }) + const durations = exported.flatMap((row) => { + const value = duration(row.startedAt ?? row.createdAt, row.finishedAt) + return value === null ? [] : [value] + }) + const byKind: Record = {} + for (const row of exported) byKind[row.kind] = (byKind[row.kind] ?? 0) + 1 + const byRepo: Record = {} + for (const row of exported) byRepo[row.repo] = (byRepo[row.repo] ?? 0) + 1 + console.log(JSON.stringify({ + database: dbPath, + schemaVersion: version, + windowDays: days, + dataset: { id: dataset.manifest.id, contentHash: dataset.manifest.contentHash }, + records: exported.length, + baseline: { + succeeded: succeeded.length, + failed: failed.length, + successRate: exported.length === 0 ? null : Math.round((succeeded.length / exported.length) * 1000) / 10, + prs: prs.length, + merged: merged.length, + mergeRate: prs.length === 0 ? null : Math.round((merged.length / prs.length) * 1000) / 10, + reviewAverage: scores.length === 0 ? null : Math.round((scores.reduce((sum, score) => sum + score, 0) / scores.length) * 10) / 10, + reviewCount: scores.length, + evidenceCompleteness: exported.length === 0 ? null : Math.round((evidenceComplete.length / exported.length) * 1000) / 10, + medianJobMs: median(durations), + }, + // The manifest's own split, not a second computation: `--report` and the + // exported file cannot disagree about which rows are held out. + split: { + train: dataset.manifest.split.train.length, + holdout: dataset.manifest.split.holdout.length, + holdoutProtected: dataset.manifest.holdoutProtected.length, + }, + byKind, + byRepo, + }, null, 2)) + db.close() + process.exit(0) +} + +if (summaryOnly) { + const byStatus: Record = {} + const byKind: Record = {} + for (const row of dataset.records) { + byStatus[row.status] = (byStatus[row.status] ?? 0) + 1 + byKind[row.kind] = (byKind[row.kind] ?? 0) + 1 + } + console.log(JSON.stringify({ + database: dbPath, + schemaVersion: version, + days, + records: dataset.records.length, + byStatus, + byKind, + dataset: { + id: dataset.manifest.id, + contentHash: dataset.manifest.contentHash, + split: { train: dataset.manifest.split.train.length, holdout: dataset.manifest.split.holdout.length }, + }, + }, null, 2)) + db.close() + process.exit(0) +} + +const text = `${JSON.stringify({ manifest: dataset.manifest, records: dataset.records }, null, 2)}\n` +if (out !== '') { + writeFileSync(resolve(out), text) + console.error(`wrote ${dataset.records.length} records to ${resolve(out)}`) +} else { + process.stdout.write(text) +} +db.close() diff --git a/src/db.ts b/src/db.ts index 3afa0d1f..2b189e2c 100644 --- a/src/db.ts +++ b/src/db.ts @@ -133,6 +133,8 @@ export interface RepoRow { job_mode: string repo_role: string | null auto_merge: number + /** #425: 1 when the repository consented to its jobs entering an evolution dataset. */ + evolution_consent: number role_checked_at: string | null plan_required: number allow_paths: string @@ -1948,9 +1950,9 @@ export class Store { this.db.exec('PRAGMA foreign_keys = ON;') this.db.exec('PRAGMA busy_timeout = 5000;') const previousVersion = Number(this.get<{ user_version: number }>('PRAGMA user_version')?.user_version ?? 0) - if (previousVersion > 39) { + if (previousVersion > 40) { this.db.close() - throw new Error(`database schema version ${previousVersion} is newer than this build supports (39)`) + throw new Error(`database schema version ${previousVersion} is newer than this build supports (40)`) } this.db.exec(SCHEMA) this.migrateColumns() @@ -1973,7 +1975,7 @@ export class Store { this.run('UPDATE repos SET role_checked_at = ? WHERE repo_role IS NOT NULL AND role_checked_at IS NULL', nowIso()) } this.migrateIdentity() - this.db.exec(`PRAGMA user_version = ${Math.max(previousVersion, 39)};`) + this.db.exec(`PRAGMA user_version = ${Math.max(previousVersion, 40)};`) } // #544 slice 3B-B2: a database written by slice B1 already has the occurrence @@ -2215,6 +2217,7 @@ export class Store { ['job_mode', "TEXT NOT NULL DEFAULT 'iterate'"], ['repo_role', 'TEXT'], ['auto_merge', 'INTEGER NOT NULL DEFAULT 0'], + ['evolution_consent', 'INTEGER NOT NULL DEFAULT 0'], ['role_checked_at', 'TEXT'], ['plan_required', 'INTEGER NOT NULL DEFAULT 0'], ['allow_paths', "TEXT NOT NULL DEFAULT ''"], @@ -4345,6 +4348,24 @@ export class Store { this.run(`UPDATE repos SET ${sets} WHERE id = ?`, ...entries.map(([, value]) => value), id) } + // #425: consent is per repository, off by default, and the only way on is an + // audited operator action. The exporter reads `evolution_consent` and nothing + // else, so a repository that never opted in cannot appear in a dataset. + setRepoEvolutionConsent(repoId: string, on: boolean, actorUserId: string | null): void { + this.run('UPDATE repos SET evolution_consent = ? WHERE id = ?', on ? 1 : 0, repoId) + this.audit(actorUserId, 'evolution.consent', `${repoId}: ${on ? 'on' : 'off'}`) + } + + listEvolutionConsent(): Array<{ repoId: string; repo: string; consent: boolean }> { + return this.all<{ id: string; owner: string; name: string; upstream_owner: string; upstream_name: string; evolution_consent: number }>( + 'SELECT id, owner, name, upstream_owner, upstream_name, evolution_consent FROM repos ORDER BY created_at DESC', + ).map((row) => ({ + repoId: row.id, + repo: `${row.upstream_owner || row.owner}/${row.upstream_name || row.name}`, + consent: row.evolution_consent === 1, + })) + } + deleteRepo(id: string): void { this.run('DELETE FROM repos WHERE id = ?', id) } diff --git a/src/lib/evolution/dataset.ts b/src/lib/evolution/dataset.ts new file mode 100644 index 00000000..4cf6d91c --- /dev/null +++ b/src/lib/evolution/dataset.ts @@ -0,0 +1,435 @@ +// #425: the dataset registry. Everything here is pure — the exporter hands in +// the rows it read and gets back a manifest and the records it must write — so +// the hash, the split and the redaction rule are testable without a database, a +// clock or a network. +// +// Two properties this module owns and the tests pin: +// - one recogniser: every exported string passes through redact(), the same +// function the rest of the platform uses; there is no second pattern list. +// - one hash: contentHash covers the selection, the split and the canonical rows, +// never the wall-clock window, so the same rows and parameters hash the same +// twice and `verifyDataset` can recompute it from the written file. + +import { createHash } from 'node:crypto' +import { redact } from '../redact.js' + +export const DATASET_SCHEMA_VERSION = 1 +export const REDACTION_VERSION = 1 + +// The job columns the export reads. A database missing one cannot produce an +// honest record, so the exporter refuses instead of exporting a partial row — +// that is what replaces the frozen `user_version > 28` check. +export const REQUIRED_JOB_COLUMNS = ['kind', 'evidence_json', 'team_stats_json'] as const + +export interface EvolutionRow { + jobId: string + repo: string + repoGoal: string + kind: string + status: string + trigger: string + goal: string + baseSha: string | null + branch: string | null + prUrl: string | null + issueUrl: string | null + reviewScore: number | null + evidence: unknown + attribution: unknown + teamStats: unknown + checkpoint: string | null + costUsd: number + createdAt: string + startedAt: string | null + finishedAt: string | null + stageMode: string | null + repoRole: string + autoMerge: boolean +} + +// What the caller asked for. The applied limit is part of the dataset's identity: +// two files with the same rows but a different truncation are not the same +// selection, and the hash has to say so. +export interface DatasetSelection { + days: number + limit: number | null +} + +// Provenance, deliberately outside the hash: when the export ran, which window it +// was asked for, and the real coverage of the rows it actually wrote. +export interface DatasetWindow { + from: string + to: string + oldestRowAt: string | null + newestRowAt: string | null +} + +export interface DatasetManifest { + id: string + schemaVersion: number + redactionVersion: number + selection: DatasetSelection + window: DatasetWindow + repositories: string[] + rowCount: number + trainRows: number + holdoutRows: number + contentHash: string + split: { train: string[]; holdout: string[] } + holdoutProtected: string[] +} + +export interface Dataset { + manifest: DatasetManifest + records: EvolutionRow[] +} + +export interface BuildOptions { + days: number + limit?: number + now?: Date +} + +export interface HoldoutLeakRefusal { + ok: false + code: 'HOLDOUT_LEAK' + protected: string +} + +export interface HoldoutLeakOk { + ok: true +} + +export type HoldoutLeakVerdict = HoldoutLeakOk | HoldoutLeakRefusal + +export type ExportRefusalCode = 'MISSING_JOB_COLUMNS' | 'NO_TRAINING_ROWS' + +// A named refusal, so a caller can say which rule stopped it rather than match +// on prose. The exporter prints the code and the sentence. +export class ExportRefusal extends Error { + constructor(readonly code: ExportRefusalCode, message: string) { + super(`${code}: ${message}`) + this.name = 'ExportRefusal' + } +} + +export type VerifyCode = + | 'MALFORMED_DATASET' + | 'SPLIT_MISMATCH' + | 'MANIFEST_MISMATCH' + | 'CONTENT_HASH_MISMATCH' + | 'DATASET_ID_MISMATCH' + +export type VerifyVerdict = + | { ok: true; manifest: DatasetManifest } + | { ok: false; code: VerifyCode; detail: string } + +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== 'object') return JSON.stringify(value) + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]` + const entries = Object.entries(value as Record).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`).join(',')}}` +} + +function redactValue(value: unknown): unknown { + if (typeof value === 'string') return redact(value) + if (Array.isArray(value)) return value.map(redactValue) + if (value !== null && typeof value === 'object') { + return Object.fromEntries(Object.entries(value as Record).map(([key, item]) => [key, redactValue(item)])) + } + return value +} + +function redactRow(row: EvolutionRow): EvolutionRow { + return { + ...row, + repo: redact(row.repo), + repoGoal: redact(row.repoGoal), + kind: redact(row.kind), + status: redact(row.status), + trigger: redact(row.trigger), + goal: redact(row.goal), + baseSha: row.baseSha === null ? null : redact(row.baseSha), + branch: row.branch === null ? null : redact(row.branch), + prUrl: row.prUrl === null ? null : redact(row.prUrl), + issueUrl: row.issueUrl === null ? null : redact(row.issueUrl), + evidence: redactValue(row.evidence), + attribution: redactValue(row.attribution), + teamStats: redactValue(row.teamStats), + checkpoint: row.checkpoint === null ? null : redact(row.checkpoint), + createdAt: redact(row.createdAt), + startedAt: row.startedAt === null ? null : redact(row.startedAt), + finishedAt: row.finishedAt === null ? null : redact(row.finishedAt), + stageMode: row.stageMode === null ? null : redact(row.stageMode), + repoRole: redact(row.repoRole), + } +} + +// One canonical order for the records and one for the split. `finishedAt` is the +// newest answer a row has; a row still running falls back to its start and then to +// creation, so ordering never depends on the input array. +function finishedKey(row: EvolutionRow): string { + return row.finishedAt ?? row.startedAt ?? row.createdAt +} + +function compareRows(a: EvolutionRow, b: EvolutionRow): number { + if (a.repo !== b.repo) return a.repo < b.repo ? -1 : 1 + const aKey = finishedKey(a) + const bKey = finishedKey(b) + if (aKey !== bKey) return aKey < bKey ? -1 : 1 + return a.jobId < b.jobId ? -1 : a.jobId > b.jobId ? 1 : 0 +} + +// The real coverage of the rows in the file, independent of the requested window. +function rowExtent(rows: readonly EvolutionRow[]): { oldest: string | null; newest: string | null } { + let oldest = Number.POSITIVE_INFINITY + let newest = Number.NEGATIVE_INFINITY + for (const row of rows) { + const value = Date.parse(finishedKey(row)) + if (!Number.isFinite(value)) continue + if (value < oldest) oldest = value + if (value > newest) newest = value + } + if (oldest === Number.POSITIVE_INFINITY) return { oldest: null, newest: null } + return { oldest: new Date(oldest).toISOString(), newest: new Date(newest).toISOString() } +} + +// The holdout entry a prompt must not contain: the pull request when the row has +// one, otherwise the job id. Both are stable and both name the row. +export function holdoutEntry(row: EvolutionRow): string { + const number = row.prUrl === null ? null : /\/pull\/(\d+)\b/.exec(row.prUrl)?.[1] ?? null + return number === null ? `${row.repo}#${row.jobId}` : `${row.repo}#${number}` +} + +// A repository that appears in the newest tail puts all of its rows on the +// holdout side, so no repository is ever on both sides. The tail is 20% of rows +// by the canonical newest-first order; ties break on jobId, so the same input +// always yields the same split. Repository-disjointness wins over the exact time +// cut: the tail only chooses which repositories are held out, so the holdout can +// exceed 20% and a holdout row can be older than a train row. +function splitRows(rows: EvolutionRow[]): { train: EvolutionRow[]; holdout: EvolutionRow[] } { + if (rows.length === 0) return { train: [], holdout: [] } + const newestFirst = [...rows].sort((a, b) => { + const aKey = finishedKey(a) + const bKey = finishedKey(b) + if (aKey !== bKey) return aKey < bKey ? 1 : -1 + return a.jobId < b.jobId ? -1 : a.jobId > b.jobId ? 1 : 0 + }) + const holdoutCount = Math.max(1, Math.floor(rows.length * 0.2)) + const tailRepos = new Set(newestFirst.slice(0, holdoutCount).map((row) => row.repo)) + const holdout = rows.filter((row) => tailRepos.has(row.repo)) + const train = rows.filter((row) => !tailRepos.has(row.repo)) + return { train, holdout } +} + +type HashedManifest = Pick< + DatasetManifest, + 'schemaVersion' | 'redactionVersion' | 'selection' | 'repositories' | 'rowCount' | 'trainRows' | 'holdoutRows' | 'split' | 'holdoutProtected' +> + +// The exact object the hash covers. `window`, `id` and `contentHash` are not in +// it: the first is provenance, the other two are derived from this hash. +function hashedProjection(manifest: HashedManifest): HashedManifest { + return { + schemaVersion: manifest.schemaVersion, + redactionVersion: manifest.redactionVersion, + selection: { days: manifest.selection.days, limit: manifest.selection.limit }, + repositories: [...manifest.repositories], + rowCount: manifest.rowCount, + trainRows: manifest.trainRows, + holdoutRows: manifest.holdoutRows, + split: { train: [...manifest.split.train], holdout: [...manifest.split.holdout] }, + holdoutProtected: [...manifest.holdoutProtected], + } +} + +function contentHashOf(projection: HashedManifest, rows: readonly EvolutionRow[]): string { + return createHash('sha256') + .update(canonicalJson(hashedProjection(projection))) + .update(canonicalJson(rows)) + .digest('hex') +} + +export function buildDataset(inputRows: readonly EvolutionRow[], options: BuildOptions): Dataset { + const rows = inputRows.map(redactRow).sort(compareRows) + const selection: DatasetSelection = { days: Math.max(1, options.days), limit: options.limit ?? null } + const { train, holdout } = splitRows(rows) + const extent = rowExtent(rows) + // Without an explicit `now`, the window ends at the newest row instead of the + // wall clock, so the provenance line describes the export rather than the + // moment it was run. + const now = options.now ?? (extent.newest === null ? new Date(0) : new Date(extent.newest)) + const from = new Date(now.getTime() - selection.days * 86_400_000).toISOString() + const projection: HashedManifest = { + schemaVersion: DATASET_SCHEMA_VERSION, + redactionVersion: REDACTION_VERSION, + selection, + repositories: [...new Set(rows.map((row) => row.repo))].sort(), + rowCount: rows.length, + trainRows: train.length, + holdoutRows: holdout.length, + split: { train: train.map((row) => row.jobId), holdout: holdout.map((row) => row.jobId) }, + holdoutProtected: [...new Set(holdout.map(holdoutEntry))].sort(), + } + const contentHash = contentHashOf(projection, rows) + const manifest: DatasetManifest = { + id: `ds_${contentHash.slice(0, 16)}`, + ...projection, + window: { from, to: now.toISOString(), oldestRowAt: extent.oldest, newestRowAt: extent.newest }, + contentHash, + } + return { manifest, records: rows } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((item) => typeof item === 'string') +} + +function sameStringArray(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((item, index) => item === right[index]) +} + +function malformed(detail: string): VerifyVerdict { + return { ok: false, code: 'MALFORMED_DATASET', detail } +} + +function parseRecords(value: unknown): EvolutionRow[] | null { + if (!Array.isArray(value)) return null + const records: EvolutionRow[] = [] + for (const item of value) { + if (!isRecord(item)) return null + const nullable = (field: string): boolean => item[field] === null || typeof item[field] === 'string' + if (typeof item.jobId !== 'string' || typeof item.repo !== 'string' || typeof item.createdAt !== 'string') return null + if (!nullable('startedAt') || !nullable('finishedAt')) return null + records.push(item as unknown as EvolutionRow) + } + return records +} + +// Recompute the artifact from its own bytes. The split is recomputed from the +// records, so moving a job id between sides is refused even if the attacker also +// recomputes the hash; the id and hash are then compared to the declared ones. +export function verifyDataset(payload: unknown): VerifyVerdict { + if (!isRecord(payload) || !isRecord(payload.manifest) || !Array.isArray(payload.records)) { + return malformed('payload must be { manifest, records }') + } + const manifest = payload.manifest + const selection = manifest.selection + const window = manifest.window + const split = manifest.split + if ( + typeof manifest.id !== 'string' || + typeof manifest.schemaVersion !== 'number' || + typeof manifest.redactionVersion !== 'number' || + typeof manifest.contentHash !== 'string' || + typeof manifest.rowCount !== 'number' || + typeof manifest.trainRows !== 'number' || + typeof manifest.holdoutRows !== 'number' || + !isRecord(selection) || + typeof selection.days !== 'number' || + !(selection.limit === null || typeof selection.limit === 'number') || + !isRecord(window) || + !['from', 'to'].every((field) => typeof window[field] === 'string') || + !['oldestRowAt', 'newestRowAt'].every((field) => window[field] === null || typeof window[field] === 'string') || + !isStringArray(manifest.repositories) || + !isRecord(split) || + !isStringArray(split.train) || + !isStringArray(split.holdout) || + !isStringArray(manifest.holdoutProtected) + ) { + return malformed('manifest fields are missing or have the wrong type') + } + const records = parseRecords(payload.records) + if (records === null) return malformed('every record needs jobId, repo, createdAt and nullable startedAt/finishedAt') + + const rows = [...records].sort(compareRows) + const expected = splitRows(rows) + if ( + !sameStringArray(expected.train.map((row) => row.jobId), split.train) || + !sameStringArray(expected.holdout.map((row) => row.jobId), split.holdout) || + !sameStringArray([...new Set(expected.holdout.map(holdoutEntry))].sort(), manifest.holdoutProtected) + ) { + return { ok: false, code: 'SPLIT_MISMATCH', detail: 'the split does not match the repository-disjoint rule for these records' } + } + + const projection: HashedManifest = { + schemaVersion: manifest.schemaVersion, + redactionVersion: manifest.redactionVersion, + selection: { days: selection.days, limit: selection.limit as number | null }, + repositories: manifest.repositories, + rowCount: manifest.rowCount, + trainRows: manifest.trainRows, + holdoutRows: manifest.holdoutRows, + split: { train: split.train, holdout: split.holdout }, + holdoutProtected: manifest.holdoutProtected, + } + const repositories = [...new Set(rows.map((row) => row.repo))].sort() + if ( + !sameStringArray(repositories, manifest.repositories) || + manifest.rowCount !== rows.length || + manifest.trainRows !== expected.train.length || + manifest.holdoutRows !== expected.holdout.length + ) { + return { ok: false, code: 'MANIFEST_MISMATCH', detail: 'rowCount, trainRows, holdoutRows or repositories disagree with the records' } + } + + const contentHash = contentHashOf(projection, rows) + if (contentHash !== manifest.contentHash) { + return { ok: false, code: 'CONTENT_HASH_MISMATCH', detail: `recomputed ${contentHash}` } + } + const id = `ds_${contentHash.slice(0, 16)}` + if (id !== manifest.id) return { ok: false, code: 'DATASET_ID_MISMATCH', detail: `recomputed ${id}` } + + return { + ok: true, + manifest: { + id, + ...projection, + window: { + from: window.from as string, + to: window.to as string, + oldestRowAt: window.oldestRowAt as string | null, + newestRowAt: window.newestRowAt as string | null, + }, + contentHash, + }, + } +} + +// A prompt that repeats a holdout entry is a leak. Case and any whitespace run +// are ignored, including the space some writers put before `#`, so `a/r #12` is +// the same entry as `A/R#12`; the refusal names the entry so the operator can +// find the row. +function normalize(value: string): string { + return value.toLowerCase().replace(/\s*#\s*/g, '#').replace(/\s+/g, ' ').trim() +} + +// #427 calls this before a promotion; the function ships here so the call site +// has nothing to invent. +export function checkHoldoutLeak(prompt: string, manifest: DatasetManifest): HoldoutLeakVerdict { + const text = normalize(prompt) + for (const entry of manifest.holdoutProtected) { + if (text.includes(normalize(entry))) return { ok: false, code: 'HOLDOUT_LEAK', protected: entry } + } + return { ok: true } +} + +// Replaces the frozen `user_version > 28` refusal and refuses a dataset nobody +// can train on. The version is accepted so the call site reads as "these columns +// at this schema"; the rule is the columns, because a newer schema that still has +// them exports a complete record. +export function assertExportable(columns: Iterable, version: number, trainRows?: number): void { + void version + const present = new Set(columns) + const missing = REQUIRED_JOB_COLUMNS.filter((column) => !present.has(column)) + if (missing.length > 0) { + throw new ExportRefusal('MISSING_JOB_COLUMNS', `database is missing the job columns this export reads: ${missing.join(', ')}; start the current Fledge once before exporting`) + } + if (trainRows === 0) { + throw new ExportRefusal('NO_TRAINING_ROWS', 'the split holds out every exported row, so there is no training side; consent a second repository or export more history') + } +} diff --git a/src/lib/evolution/export.ts b/src/lib/evolution/export.ts new file mode 100644 index 00000000..577c4492 --- /dev/null +++ b/src/lib/evolution/export.ts @@ -0,0 +1,58 @@ +// #425: the one row loader the exporter and its tests share. It is a query, not +// a rule — the consent filter lives in the SQL so a repository that did not opt +// in cannot reach `buildDataset` even by accident. +import type { DatabaseSync } from 'node:sqlite' +import type { EvolutionRow } from './dataset.js' + +export interface LoadOptions { + since: string + limit: number +} + +type Row = Record + +function parse(value: unknown): unknown { + if (typeof value !== 'string' || value === '') return null + try { return JSON.parse(value) } catch { return null } +} + +export function loadEvolutionRows(db: DatabaseSync, options: LoadOptions): EvolutionRow[] { + const rows = db.prepare(` + SELECT j.id, j.kind, j.status, j.trigger, j.goal, j.branch, j.pr_url, j.issue_url, + j.review_score, j.evidence_json, j.attribution_json, j.team_stats_json, + j.checkpoint, j.base_sha, j.cost_usd, j.created_at, j.started_at, j.finished_at, + r.owner, r.name, r.upstream_owner, r.upstream_name, r.goal AS repo_goal, + r.stage_mode, r.repo_role, r.auto_merge + FROM jobs j + JOIN repos r ON r.id = j.repo_id + WHERE j.created_at >= ? AND r.evolution_consent = 1 + ORDER BY j.created_at DESC + LIMIT ? + `).all(options.since, options.limit) as Row[] + + return rows.map((row) => ({ + jobId: String(row.id), + repo: `${String(row.upstream_owner || row.owner)}/${String(row.upstream_name || row.name)}`, + repoGoal: String(row.repo_goal ?? ''), + kind: String(row.kind), + status: String(row.status), + trigger: String(row.trigger ?? ''), + goal: String(row.goal ?? ''), + baseSha: row.base_sha === null ? null : String(row.base_sha), + branch: row.branch === null ? null : String(row.branch), + prUrl: row.pr_url === null ? null : String(row.pr_url), + issueUrl: row.issue_url === null ? null : String(row.issue_url), + reviewScore: typeof row.review_score === 'number' ? row.review_score : null, + evidence: parse(row.evidence_json), + attribution: parse(row.attribution_json), + teamStats: parse(row.team_stats_json), + checkpoint: row.checkpoint === null ? null : String(row.checkpoint), + costUsd: Number(row.cost_usd ?? 0), + createdAt: String(row.created_at), + startedAt: row.started_at === null ? null : String(row.started_at), + finishedAt: row.finished_at === null ? null : String(row.finished_at), + stageMode: row.stage_mode === null ? null : String(row.stage_mode), + repoRole: String(row.repo_role ?? ''), + autoMerge: row.auto_merge === 1, + })) +} diff --git a/test/db.test.ts b/test/db.test.ts index 9564e410..bcdd0482 100644 --- a/test/db.test.ts +++ b/test/db.test.ts @@ -151,9 +151,9 @@ test('the role backfill runs once and future schema versions are refused', () => const reopened = new Store(path) assert.equal(reopened.getRepo(repo.id)?.repo_role, 'maintainer') assert.notEqual(reopened.getRepo(repo.id)?.role_checked_at, null) - // #544 slice 5 moved the build's schema to 39, so the first refused version - // is 40. - reopened.db.exec('PRAGMA user_version = 40;') + // #425 moved the build's schema to 40, so the first refused version + // is 41. + reopened.db.exec('PRAGMA user_version = 41;') reopened.close() assert.throws(() => new Store(path), /newer than this build/) diff --git a/test/evolutionDataset.test.ts b/test/evolutionDataset.test.ts new file mode 100644 index 00000000..c4612cdf --- /dev/null +++ b/test/evolutionDataset.test.ts @@ -0,0 +1,237 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { assertExportable, buildDataset, checkHoldoutLeak, REQUIRED_JOB_COLUMNS, type EvolutionRow } from '../src/lib/evolution/dataset.js' +import { makeUser } from './helpers.js' + +function row(input: Partial & { jobId: string; repo: string; finishedAt: string }): EvolutionRow { + return { + repoGoal: '', + kind: 'iterate', + status: 'succeeded', + trigger: 'manual', + goal: 'tighten the parser', + baseSha: null, + branch: null, + prUrl: null, + issueUrl: null, + reviewScore: null, + evidence: null, + attribution: null, + teamStats: null, + checkpoint: null, + costUsd: 0, + createdAt: input.finishedAt, + startedAt: input.finishedAt, + stageMode: null, + repoRole: 'maintainer', + autoMerge: false, + ...input, + } +} + +const NOW = new Date('2026-11-15T00:00:00.000Z') +const OPTIONS = { days: 90, now: NOW } + +function spread(): EvolutionRow[] { + const rows: EvolutionRow[] = [] + for (let index = 0; index < 5; index += 1) { + rows.push(row({ jobId: `a_${index}`, repo: 'a/r', finishedAt: `2026-09-0${index + 1}T00:00:00.000Z` })) + } + for (let index = 0; index < 5; index += 1) { + rows.push(row({ jobId: `b_${index}`, repo: 'b/r', finishedAt: `2026-11-0${index + 1}T00:00:00.000Z` })) + } + return rows +} + +test('every exported string passes through the one recogniser and the row is kept', () => { + const dataset = buildDataset([ + row({ + jobId: 'job_secret', + repo: 'a/r', + finishedAt: '2026-11-01T00:00:00.000Z', + goal: `use ${'ghp_'}${'a'.repeat(36)} to read the repo`, + evidence: { summary: `the callback answered with ${'flg_'}${'a'.repeat(22)}` }, + }), + ], OPTIONS) + const text = JSON.stringify(dataset) + assert.equal(/ghp_[A-Za-z0-9]{20,}|flg_[A-Za-z0-9_-]{16,}/.test(text), false, 'a secret survived the export') + assert.equal(dataset.records.length, 1, 'the row must be redacted, not dropped') + assert.match(text, /\[redacted-github-token\]/) + assert.match(text, /\[redacted-fledge-token\]/) +}) + +test('the same rows give the same id and hash, in any input order', () => { + const rows = spread() + const first = buildDataset(rows, OPTIONS) + const second = buildDataset([...rows].reverse(), OPTIONS) + assert.equal(first.manifest.contentHash, second.manifest.contentHash) + assert.equal(first.manifest.id, second.manifest.id) + assert.equal(first.manifest.contentHash.length, 64) + assert.equal(first.manifest.id, `ds_${first.manifest.contentHash.slice(0, 16)}`) +}) + +test('without an explicit now the window ends at the newest row, so two exports match', () => { + // The CLI does not pass `now`; if the default were the wall clock, two exports + // of the same rows would carry different hashes and no reader could check that + // they describe the same dataset. + const rows = spread() + const first = buildDataset(rows, { days: 90 }) + const second = buildDataset([...rows].reverse(), { days: 90 }) + assert.equal(first.manifest.window.to, '2026-11-05T00:00:00.000Z') + assert.equal(first.manifest.window.from, '2026-08-07T00:00:00.000Z') + assert.equal(first.manifest.contentHash, second.manifest.contentHash) + assert.equal(first.manifest.id, second.manifest.id) +}) + +test('a changed row and a changed window both change the hash', () => { + const rows = spread() + const base = buildDataset(rows, OPTIONS) + const changedRow = buildDataset(rows.map((item) => item.jobId === 'a_0' ? { ...item, goal: 'something else entirely' } : item), OPTIONS) + assert.notEqual(base.manifest.contentHash, changedRow.manifest.contentHash) + const changedWindow = buildDataset(rows, { days: 30, now: NOW }) + assert.notEqual(base.manifest.contentHash, changedWindow.manifest.contentHash) +}) + +test('the split is repository-disjoint and takes the newest 20%', () => { + // Two repositories: the newest tail names b/r, so every b/r row moves to the + // holdout even though the tail itself is two rows. That is the rule — no + // repository is on both sides — and it is why this fixture holds out five. + const dataset = buildDataset(spread(), OPTIONS) + const { train, holdout } = dataset.manifest.split + assert.equal(holdout.length, 5) + assert.equal(train.length, 5) + const trainRepos = new Set(dataset.records.filter((item) => train.includes(item.jobId)).map((item) => item.repo)) + const holdoutRepos = new Set(dataset.records.filter((item) => holdout.includes(item.jobId)).map((item) => item.repo)) + for (const repo of holdoutRepos) assert.equal(trainRepos.has(repo), false, `${repo} is on both sides of the split`) + assert.deepEqual([...holdoutRepos], ['b/r'], 'the newest tail belongs to the later repository') + assert.equal(dataset.manifest.holdoutProtected.length, holdout.length, 'every holdout row has one protected entry') + + // Ten rows in ten repositories: the tail is exactly the newest two, and no + // repository expands it. + const oneEach = Array.from({ length: 10 }, (_, index) => row({ + jobId: `j_${index}`, + repo: `o/r${index}`, + finishedAt: `2026-10-${String(index + 1).padStart(2, '0')}T00:00:00.000Z`, + })) + const single = buildDataset(oneEach, OPTIONS) + assert.equal(single.manifest.split.holdout.length, 2) + assert.equal(single.manifest.split.train.length, 8) +}) + +test('a prompt that repeats a holdout entry is refused, and the entry is named', () => { + const dataset = buildDataset([ + row({ jobId: 'abc123', repo: 'a/r', finishedAt: '2026-11-01T00:00:00.000Z', prUrl: 'https://github.com/a/r/pull/42' }), + ], OPTIONS) + const refused = checkHoldoutLeak('apply the change from a/r#42 as the holdout did', dataset.manifest) + assert.equal(refused.ok, false) + assert.equal(refused.ok === false ? refused.code : '', 'HOLDOUT_LEAK') + assert.equal(refused.ok === false ? refused.protected : '', 'a/r#42') + const clean = checkHoldoutLeak('apply the change from a/r#43 as the holdout did', dataset.manifest) + assert.equal(clean.ok, true) + const caseAndSpace = checkHoldoutLeak('Apply the change from A/R#42 as the holdout did', dataset.manifest) + assert.equal(caseAndSpace.ok, false, 'the match is case-insensitive and whitespace-collapsed') +}) + +test('the frozen schema refusal is gone: the guard reads the columns it needs', () => { + assert.doesNotThrow(() => assertExportable(['kind', 'evidence_json', 'team_stats_json', 'goal'], 38)) + assert.throws(() => assertExportable(['kind', 'team_stats_json'], 38), /evidence_json/) + assert.throws(() => assertExportable(['kind', 'evidence_json'], 38), /team_stats_json/) + assert.equal(REQUIRED_JOB_COLUMNS.length, 3) +}) + +// The consent gate, end to end: a real Store at a temp path, the exporter's own +// row loader, and one consented plus one unconsented repository. The unconsented +// one must be absent from every side of the split and from the manifest. +test('an unconsented repository is absent from the dataset, and a consented one is present', async () => { + const { mkdtempSync, rmSync } = await import('node:fs') + const { tmpdir } = await import('node:os') + const { join } = await import('node:path') + const { DatabaseSync } = await import('node:sqlite') + const { Store } = await import('../src/db.js') + const { loadEvolutionRows } = await import('../src/lib/evolution/export.js') + const dir = mkdtempSync(join(tmpdir(), 'fledge-evolution-')) + const path = join(dir, 'fledge.db') + const store = new Store(path) + try { + const user = makeUser(store, { login: 'evo', admin: true, superadmin: true, status: 'approved' }) + const consented = store.createRepo({ id: 'repo_on', userId: user.id, owner: 'o', name: 'on', defaultBranch: 'main', tokenEnc: 'x', permission: 'admin', repoRole: 'maintainer', goal: '', autoIterate: false, intervalMinutes: 1440 }) + const quiet = store.createRepo({ id: 'repo_off', userId: user.id, owner: 'o', name: 'off', defaultBranch: 'main', tokenEnc: 'x', permission: 'admin', repoRole: 'maintainer', goal: '', autoIterate: false, intervalMinutes: 1440 }) + const createdAt = new Date().toISOString() + store.createJob({ id: 'job_on', repoId: consented.id, userId: user.id, trigger: 'manual', goal: 'on goal', callbackHash: 'h1', platformTokenHash: 'p1', createdAt }) + store.createJob({ id: 'job_off', repoId: quiet.id, userId: user.id, trigger: 'manual', goal: 'off goal', callbackHash: 'h2', platformTokenHash: 'p2', createdAt }) + store.setRepoEvolutionConsent(consented.id, true, user.id) + assert.equal(store.listEvolutionConsent().find((row) => row.repoId === consented.id)?.consent, true) + assert.equal(store.listEvolutionConsent().find((row) => row.repoId === quiet.id)?.consent, false) + assert.equal(store.listAudit(20).some((row) => row.action === 'evolution.consent'), true, 'the consent change must be audited') + + const reader = new DatabaseSync(path, { readOnly: true }) + try { + const rows = loadEvolutionRows(reader, { since: new Date(Date.now() - 86_400_000).toISOString(), limit: 100 }) + assert.deepEqual(rows.map((row) => row.jobId), ['job_on']) + const dataset = buildDataset(rows, { days: 1, now: new Date() }) + assert.deepEqual(dataset.manifest.repositories, ['o/on']) + // One row is one row: the split's minimum holdout is a single row, so the + // consented row is held out and the unconsented one appears nowhere. + assert.deepEqual([...dataset.manifest.split.train, ...dataset.manifest.split.holdout], ['job_on']) + assert.deepEqual(dataset.manifest.holdoutProtected, ['o/on#job_on']) + assert.equal(dataset.manifest.split.train.length + dataset.manifest.split.holdout.length, dataset.manifest.rowCount) + } finally { + reader.close() + } + } finally { + store.close() + rmSync(dir, { recursive: true, force: true }) + } +}) + +test('the manifest records the effective split sizes so nobody counts arrays', () => { + const dataset = buildDataset(spread(), OPTIONS) + assert.equal(dataset.manifest.trainRows, 5) + assert.equal(dataset.manifest.holdoutRows, 5) + assert.equal(dataset.manifest.trainRows + dataset.manifest.holdoutRows, dataset.manifest.rowCount) +}) + +test('a split that holds out every row is refused by name', () => { + const solo = Array.from({ length: 3 }, (_, index) => row({ + jobId: `s_${index}`, + repo: 'solo/r', + finishedAt: `2026-11-0${index + 1}T00:00:00.000Z`, + })) + const dataset = buildDataset(solo, OPTIONS) + assert.equal(dataset.manifest.trainRows, 0) + assert.throws( + () => assertExportable(['kind', 'evidence_json', 'team_stats_json'], 38, dataset.manifest.trainRows), + /NO_TRAINING_ROWS/, + ) + assert.doesNotThrow(() => assertExportable(['kind', 'evidence_json', 'team_stats_json'], 38, 1)) +}) + +test('the manifest records the selection and the exported coverage', () => { + const bounded = buildDataset(spread(), { days: 90, limit: 500 }) + assert.deepEqual(bounded.manifest.selection, { days: 90, limit: 500 }) + assert.equal(bounded.manifest.window.oldestRowAt, '2026-09-01T00:00:00.000Z') + assert.equal(bounded.manifest.window.newestRowAt, '2026-11-05T00:00:00.000Z') + const unbounded = buildDataset(spread(), OPTIONS) + assert.deepEqual(unbounded.manifest.selection, { days: 90, limit: null }) + assert.equal(unbounded.manifest.window.oldestRowAt, '2026-09-01T00:00:00.000Z') +}) + +test('the hash covers the rows and the selection, not the wall-clock window', () => { + const rows = spread() + const early = buildDataset(rows, { days: 90, limit: 500, now: new Date('2026-11-15T00:00:00.000Z') }) + const late = buildDataset(rows, { days: 90, limit: 500, now: new Date('2026-12-01T00:00:00.000Z') }) + assert.notEqual(early.manifest.window.to, late.manifest.window.to) + assert.equal(early.manifest.contentHash, late.manifest.contentHash) + assert.equal(early.manifest.id, late.manifest.id) + const wider = buildDataset(rows, { days: 90, limit: 1000 }) + assert.notEqual(early.manifest.contentHash, wider.manifest.contentHash, 'the applied limit is part of the identity') +}) + +test('a holdout entry with a space before the number is still refused', () => { + const dataset = buildDataset([ + row({ jobId: 'abc123', repo: 'a/r', finishedAt: '2026-11-01T00:00:00.000Z', prUrl: 'https://github.com/a/r/pull/42' }), + ], OPTIONS) + const refused = checkHoldoutLeak('use A/R #42 as the holdout did', dataset.manifest) + assert.equal(refused.ok, false, 'a space before # must not slip past the only guard') + assert.equal(refused.ok === false ? refused.protected : '', 'a/r#42') +}) diff --git a/test/evolutionDatasetVerify.test.ts b/test/evolutionDatasetVerify.test.ts new file mode 100644 index 00000000..4087e303 --- /dev/null +++ b/test/evolutionDatasetVerify.test.ts @@ -0,0 +1,83 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildDataset, verifyDataset, type EvolutionRow } from '../src/lib/evolution/dataset.js' + +function row(input: Partial & { jobId: string; repo: string; finishedAt: string }): EvolutionRow { + return { + repoGoal: '', + kind: 'iterate', + status: 'succeeded', + trigger: 'manual', + goal: 'tighten the parser', + baseSha: null, + branch: null, + prUrl: null, + issueUrl: null, + reviewScore: null, + evidence: null, + attribution: null, + teamStats: null, + checkpoint: null, + costUsd: 0, + createdAt: input.finishedAt, + startedAt: input.finishedAt, + stageMode: null, + repoRole: 'maintainer', + autoMerge: false, + ...input, + } +} + +const OPTIONS = { days: 90, now: new Date('2026-11-15T00:00:00.000Z') } + +function spread(): EvolutionRow[] { + const rows: EvolutionRow[] = [] + for (let index = 0; index < 5; index += 1) { + rows.push(row({ jobId: `a_${index}`, repo: 'a/r', finishedAt: `2026-09-0${index + 1}T00:00:00.000Z` })) + } + for (let index = 0; index < 5; index += 1) { + rows.push(row({ jobId: `b_${index}`, repo: 'b/r', finishedAt: `2026-11-0${index + 1}T00:00:00.000Z` })) + } + return rows +} + +test('a written dataset verifies by recomputing the hash and the id', () => { + const payload = JSON.parse(JSON.stringify(buildDataset(spread(), OPTIONS))) + const verdict = verifyDataset(payload) + assert.equal(verdict.ok, true) + assert.equal(verdict.ok === true ? verdict.manifest.id : '', payload.manifest.id) +}) + +test('a moved split id is refused as SPLIT_MISMATCH even before the hash is consulted', () => { + const payload = JSON.parse(JSON.stringify(buildDataset(spread(), OPTIONS))) as ReturnType + const moved = payload.manifest.split.holdout.pop() + assert.ok(moved) + payload.manifest.split.train.push(moved) + const verdict = verifyDataset(payload) + assert.equal(verdict.ok, false) + assert.equal(verdict.ok === false ? verdict.code : 'ok', 'SPLIT_MISMATCH') +}) + +test('an edited content hash is refused as CONTENT_HASH_MISMATCH', () => { + const payload = JSON.parse(JSON.stringify(buildDataset(spread(), OPTIONS))) as ReturnType + payload.manifest.contentHash = `${payload.manifest.contentHash[0] === 'a' ? 'b' : 'a'}${payload.manifest.contentHash.slice(1)}` + const verdict = verifyDataset(payload) + assert.equal(verdict.ok, false) + assert.equal(verdict.ok === false ? verdict.code : 'ok', 'CONTENT_HASH_MISMATCH') +}) + +test('an edited dataset id is refused as DATASET_ID_MISMATCH', () => { + const payload = JSON.parse(JSON.stringify(buildDataset(spread(), OPTIONS))) as ReturnType + payload.manifest.id = `ds_${'0'.repeat(16)}` + const verdict = verifyDataset(payload) + assert.equal(verdict.ok, false) + assert.equal(verdict.ok === false ? verdict.code : 'ok', 'DATASET_ID_MISMATCH') +}) + +test('a payload that is not a dataset is refused as MALFORMED_DATASET', () => { + for (const payload of [null, {}, { manifest: {}, records: [] }, { manifest: { records: [] } }]) { + const verdict = verifyDataset(payload) + assert.equal(verdict.ok, false) + assert.equal(verdict.ok === false ? verdict.code : 'ok', 'MALFORMED_DATASET') + } +}) diff --git a/test/scheduleLeases.test.ts b/test/scheduleLeases.test.ts index 98e59352..e070aa92 100644 --- a/test/scheduleLeases.test.ts +++ b/test/scheduleLeases.test.ts @@ -45,11 +45,11 @@ function claim(h: { store: Store; repoId: string; scheduleId: string }, token: s }) } -test('the occurrence row carries attempts and a lease, at user_version 39', () => { +test('the occurrence row carries attempts and a lease, at user_version 40', () => { const store = new Store(':memory:') try { const version = store.get<{ user_version: number }>('PRAGMA user_version') - assert.equal(Number(version?.user_version), 39, 'the grace column of slice 5 moves the refusal boundary from 38 to 39') + assert.equal(Number(version?.user_version), 40, 'the evolution consent column moves the refusal boundary from 39 to 40') const columns = store.all<{ name: string }>('PRAGMA table_info(schedule_occurrences)').map((column) => column.name) for (const column of ['attempts', 'lease_token', 'lease_expires_at']) { diff --git a/test/scheduleOccurrences.test.ts b/test/scheduleOccurrences.test.ts index 779c73cc..43301ecd 100644 --- a/test/scheduleOccurrences.test.ts +++ b/test/scheduleOccurrences.test.ts @@ -56,11 +56,11 @@ function audits(store: Store, action: string): string[] { // -- the table ------------------------------------------------------------- -test('the occurrence table is part of the schema at user_version 39', () => { +test('the occurrence table is part of the schema at user_version 40', () => { const store = new Store(':memory:') try { const version = store.get<{ user_version: number }>('PRAGMA user_version') - assert.equal(Number(version?.user_version), 39, 'the grace column of slice 5 moves the refusal boundary from 38 to 39') + assert.equal(Number(version?.user_version), 40, 'the evolution consent column moves the refusal boundary from 39 to 40') const table = store.get<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'schedule_occurrences'") assert.equal(table?.name, 'schedule_occurrences') diff --git a/test/scheduleSchema.test.ts b/test/scheduleSchema.test.ts index 00170ba9..cdaeda9a 100644 --- a/test/scheduleSchema.test.ts +++ b/test/scheduleSchema.test.ts @@ -7,11 +7,11 @@ import { makeUser } from './helpers.js' // the version boundary, the columns the planner reads, and the one-row-per- // repository rule the tick relies on. -test('the schedule table is part of the schema at user_version 39', () => { +test('the schedule table is part of the schema at user_version 40', () => { const store = new Store(':memory:') try { const version = store.get<{ user_version: number }>('PRAGMA user_version') - assert.equal(Number(version?.user_version), 39, 'a new column moves the refusal boundary: 36 the schedule, 37 the occurrence, 38 its lease, 39 its grace') + assert.equal(Number(version?.user_version), 40, 'each schema change moves the refusal boundary: 36 the schedule, 37 the occurrence, 38 its lease, 39 its grace, 40 evolution consent') const table = store.get<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'schedules'") assert.equal(table?.name, 'schedules')