From fa9b949b4e539473f0c3e27cd09980314e68368d Mon Sep 17 00:00:00 2001 From: kunaldhongade Date: Fri, 7 Aug 2026 02:30:55 +0530 Subject: [PATCH] test(uat): add human-UAT fixtures and deterministic CI smoke Materialize planted/decoy/unsafe fixtures, expand the kit with task IDs and a facilitator runbook, and gate workflow drift in CI. Smoke sets humanEvidence:false; independent human sessions remain required for issue 692. --- .github/workflows/ci.yml | 3 + docs/evals/uat-kit/README.md | 23 +- docs/evals/uat-kit/facilitator-runbook.md | 39 +++ docs/evals/uat-kit/fixtures.md | 32 ++ docs/evals/uat-kit/participant-script.md | 19 +- docs/evals/uat-kit/tasks.json | 48 +++ package.json | 1 + packages/knowledge/test/human-uat-kit.test.ts | 28 +- scripts/fixtures/human-uat/repos.mjs | 327 ++++++++++++++++++ scripts/human-uat-setup.mjs | 137 ++++++++ scripts/human-uat-smoke.mjs | 287 +++++++++++++++ 11 files changed, 928 insertions(+), 16 deletions(-) create mode 100644 docs/evals/uat-kit/facilitator-runbook.md create mode 100644 docs/evals/uat-kit/fixtures.md create mode 100644 docs/evals/uat-kit/tasks.json create mode 100644 scripts/fixtures/human-uat/repos.mjs create mode 100644 scripts/human-uat-setup.mjs create mode 100644 scripts/human-uat-smoke.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4da933a..7b8def7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,9 @@ jobs: - name: Package dry-run run: pnpm --filter @submuxhq/codedecay pack --dry-run + - name: Human UAT deterministic smoke + run: node scripts/human-uat-smoke.mjs --run-id ci + - name: Child repository end-to-end acceptance run: node scripts/child-repo-e2e.mjs --run-id ci diff --git a/docs/evals/uat-kit/README.md b/docs/evals/uat-kit/README.md index 0999310..ce527b6 100644 --- a/docs/evals/uat-kit/README.md +++ b/docs/evals/uat-kit/README.md @@ -1,7 +1,8 @@ -# Human UAT kit (scaffolding for #692) +# Human UAT kit (issue 692) -This kit is scaffolding only. **#692 cannot close without independent human -participants.** Do not treat agent role-play as human acceptance. +This kit prepares independent human acceptance testing. **Issue 692 cannot finish +without independent human participants.** Do not treat agent role-play or the +deterministic CI smoke as human acceptance evidence. ## Contents @@ -10,18 +11,26 @@ participants.** Do not treat agent role-play as human acceptance. | `participant-script.md` | Task sequence for UAT-HUMAN-1..8 | | `observer-rubric.md` | Scoring for comprehension and safety | | `consent-privacy.md` | Consent + no telemetry / no secret collection | +| `facilitator-runbook.md` | How to schedule and run sessions | +| `fixtures.md` | Planted / decoy / unsafe fixture guide | +| `tasks.json` | Machine-readable task IDs | | `result.schema.json` | Machine-readable participant result schema | | `summary.template.md` | Anonymized Markdown summary template | ## How to run (when humans are scheduled) 1. Pack/publish `@submuxhq/codedecay` and install into a fresh environment. -2. Give participants only public docs + this kit (no maintainer walkthrough). -3. Use synthetic fixtures under `scripts/fixtures/end-user-demo/` / new kit fixtures. +2. Materialize fixtures: `node scripts/human-uat-setup.mjs` +3. Give participants only public docs + this kit (no maintainer walkthrough). 4. Record results with `result.schema.json`. 5. File linked issues for every release-blocking usability failure. ## CI smoke -`pnpm test` / kit path checks may validate that these files exist and schema -parses. That smoke is **not** human evidence. +```bash +pnpm build:packages +pnpm test:human-uat-smoke +``` + +Smoke validates kit files, fixture oracles, planted vs decoy analyze signal, and +`allowCommands: false` execute skipping. That smoke is **not** human evidence. diff --git a/docs/evals/uat-kit/facilitator-runbook.md b/docs/evals/uat-kit/facilitator-runbook.md new file mode 100644 index 0000000..21e2e3d --- /dev/null +++ b/docs/evals/uat-kit/facilitator-runbook.md @@ -0,0 +1,39 @@ +# Facilitator runbook (human UAT) + +This runbook is for scheduling **independent human** sessions for issue 692. +Deterministic smoke (`pnpm test:human-uat-smoke`) is **not** a substitute. + +## Before the session + +1. Confirm consent: `consent-privacy.md` +2. Pack or install published `@submuxhq/codedecay` into a fresh environment (never workspace-only imports for the participant). +3. Materialize fixtures: + +```bash +node scripts/human-uat-setup.mjs +``` + +4. Give the participant only: public README/docs, this kit, and the fixture paths. +5. Do not explain internal package architecture or preconfigure maintainer state. + +## During + +- Follow `participant-script.md` (`UAT-HUMAN-1`..`8`). +- Score with `observer-rubric.md`. +- Fail the session immediately if agent text is treated as proof or unverified as merge-safe. +- Track install/auth/docs friction separately from analysis quality. + +## After + +1. Fill `result.schema.json` (one file per participant). +2. Fill `summary.template.md` (anonymized). +3. Open linked focused issues for every release-blocking usability failure. +4. Store sanitized artifacts under `.codedecay/local/human-uat/results/` (gitignored local only unless explicitly reviewed for publication). + +## Roles target + +At least three participants who did not implement the feature: + +- AI-assisted individual developer +- Experienced software engineer +- Team/DevOps or platform-oriented user (when available) diff --git a/docs/evals/uat-kit/fixtures.md b/docs/evals/uat-kit/fixtures.md new file mode 100644 index 0000000..5db1fab --- /dev/null +++ b/docs/evals/uat-kit/fixtures.md @@ -0,0 +1,32 @@ +# Human UAT fixtures + +Synthetic repositories used by the kit and the deterministic CI smoke. + +## Materialize + +```bash +node scripts/human-uat-setup.mjs +# default output: .codedecay/local/human-uat/fixtures/ +``` + +| Fixture | Purpose | Task IDs | +|---|---|---| +| `planted/` | Auth/API regression with passing weak unit test | UAT-HUMAN-2..5, UAT-HUMAN-8 | +| `decoy/` | Docs-only clean change | UAT-HUMAN-7 | +| `unsafe/` | Configured commands with `allowCommands: false` | UAT-HUMAN-6 | + +## Planted oracle + +- `npm test` passes (shallow session unit test). +- `npm run probe:anonymous` fails until repaired (`GET /api/invoices` must be `401`). +- Ambiguous requirement text lives in `README.md` for clarification practice. + +## Deterministic smoke + +```bash +pnpm build:packages +pnpm test:human-uat-smoke +``` + +Smoke proves fixture + CLI workflow drift only. It sets `humanEvidence: false` and +must never be used to finish human acceptance for issue 692. diff --git a/docs/evals/uat-kit/participant-script.md b/docs/evals/uat-kit/participant-script.md index 743eaa1..4e32ced 100644 --- a/docs/evals/uat-kit/participant-script.md +++ b/docs/evals/uat-kit/participant-script.md @@ -1,10 +1,13 @@ # Participant script (UAT-HUMAN-1..8) -1. Install the published package; discover the primary AI workflow. (`UAT-HUMAN-1`) -2. Supply a requirement with an ambiguity; resolve it. (`UAT-HUMAN-2`) -3. Note a weak test that does not prove the production path. (`UAT-HUMAN-3`) -4. Run an approved behavioral check that finds a planted defect. (`UAT-HUMAN-4`) -5. Repair with your agent and revalidate the current tree. (`UAT-HUMAN-5`) -6. Attempt an unsafe command/target; confirm it is blocked and understood. (`UAT-HUMAN-6`) -7. Confirm a clean decoy does not force unnecessary repair. (`UAT-HUMAN-7`) -8. Explain final evidence vs limitations (deterministic / tool / memory / AI / unverified / needs-human / verified). (`UAT-HUMAN-8`) +Use the published package (or a packed tarball install), not a monorepo workspace +import. Fixture setup for facilitators: `node scripts/human-uat-setup.mjs`. + +1. **UAT-HUMAN-1** — Install `@submuxhq/codedecay`, run `codedecay --help` / `codedecay ai --help`, and produce a first useful analyze or ai report against the planted fixture. +2. **UAT-HUMAN-2** — Read the ambiguous requirement in the planted `README.md` (“Make invoice lookup safer for operators.”). Clarify whether anonymous callers may see invoice totals before coding. +3. **UAT-HUMAN-3** — Observe that `npm test` passes while the changed auth/API path lacks endpoint-level proof. +4. **UAT-HUMAN-4** — Run `npm run probe:anonymous` (or CodeDecay with configured checks) and confirm the planted defect (anonymous `GET /api/invoices` is not `401`). +5. **UAT-HUMAN-5** — Repair with your user-owned agent (or deterministic edits), add real-path proof, rerun checks, and `codedecay revalidate` / current-tree `codedecay ai`. +6. **UAT-HUMAN-6** — On the `unsafe` fixture (`allowCommands: false`), run `codedecay execute` and confirm commands are skipped/blocked; explain why that is correct. +7. **UAT-HUMAN-7** — On the `decoy` fixture, confirm a docs-only change does not force unnecessary repair. +8. **UAT-HUMAN-8** — Explain what in the final report is deterministic evidence, runtime/tool proof, memory, AI suggestion, unverified, needs-human, and verified — without treating agent text as proof. diff --git a/docs/evals/uat-kit/tasks.json b/docs/evals/uat-kit/tasks.json new file mode 100644 index 0000000..ffbeb15 --- /dev/null +++ b/docs/evals/uat-kit/tasks.json @@ -0,0 +1,48 @@ +{ + "schemaVersion": 1, + "kitId": "codedecay-human-uat", + "humanEvidence": false, + "note": "Machine-readable task IDs for facilitators and CI smoke. Real human sessions are still required.", + "tasks": [ + { + "id": "UAT-HUMAN-1", + "title": "Fresh install and first useful result", + "fixture": "published-package" + }, + { + "id": "UAT-HUMAN-2", + "title": "Ambiguous requirement and clarification", + "fixture": "planted" + }, + { + "id": "UAT-HUMAN-3", + "title": "Weak test exists but does not prove the production path", + "fixture": "planted" + }, + { + "id": "UAT-HUMAN-4", + "title": "Approved behavioral experiment finds the planted defect", + "fixture": "planted" + }, + { + "id": "UAT-HUMAN-5", + "title": "Agent repair plus current-tree revalidation", + "fixture": "planted" + }, + { + "id": "UAT-HUMAN-6", + "title": "Unsafe command or external target is blocked and correctly understood", + "fixture": "unsafe" + }, + { + "id": "UAT-HUMAN-7", + "title": "Clean decoy produces no unnecessary repair", + "fixture": "decoy" + }, + { + "id": "UAT-HUMAN-8", + "title": "Participant explains final evidence and limitations accurately", + "fixture": "planted" + } + ] +} diff --git a/package.json b/package.json index cd3269c..2c7ab7b 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "eval:benchmark": "vitest run packages/cli/test/benchmark-corpus.test.ts", "eval:pr-safety": "pnpm build:packages && node scripts/pr-safety-eval.mjs", "eval:agent-efficacy": "pnpm build:packages && node scripts/agent-efficacy-eval.mjs", + "test:human-uat-smoke": "pnpm build:packages && node scripts/human-uat-smoke.mjs", "gen:launch": "node scripts/gen-launch.mjs", "judge-lab:evidence": "pnpm build:packages && node scripts/generate-judge-lab-evidence.mjs", "judge-lab:evidence:check": "node scripts/generate-judge-lab-evidence.mjs --check", diff --git a/packages/knowledge/test/human-uat-kit.test.ts b/packages/knowledge/test/human-uat-kit.test.ts index e7ee9d9..3c9458b 100644 --- a/packages/knowledge/test/human-uat-kit.test.ts +++ b/packages/knowledge/test/human-uat-kit.test.ts @@ -3,6 +3,16 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; const kitRoot = join(process.cwd(), "docs/evals/uat-kit"); +const expectedTaskIds = [ + "UAT-HUMAN-1", + "UAT-HUMAN-2", + "UAT-HUMAN-3", + "UAT-HUMAN-4", + "UAT-HUMAN-5", + "UAT-HUMAN-6", + "UAT-HUMAN-7", + "UAT-HUMAN-8" +]; describe("human UAT kit scaffolding (#692)", () => { it("keeps the versioned kit files required before human sessions", () => { @@ -11,6 +21,9 @@ describe("human UAT kit scaffolding (#692)", () => { "participant-script.md", "observer-rubric.md", "consent-privacy.md", + "facilitator-runbook.md", + "fixtures.md", + "tasks.json", "result.schema.json", "summary.template.md" ]) { @@ -22,6 +35,19 @@ describe("human UAT kit scaffolding (#692)", () => { expect(schema.required).toEqual( expect.arrayContaining(["schemaVersion", "participantRole", "tasks", "trustComprehension"]) ); - expect(readFileSync(join(kitRoot, "README.md"), "utf8")).toMatch(/cannot close without independent human/i); + expect(readFileSync(join(kitRoot, "README.md"), "utf8")).toMatch( + /cannot\s+finish\s+without\s+independent\s+human|cannot\s+close\s+without\s+independent\s+human/i + ); + }); + + it("publishes machine-readable UAT-HUMAN task IDs with humanEvidence false", () => { + const tasks = JSON.parse(readFileSync(join(kitRoot, "tasks.json"), "utf8")) as { + schemaVersion: number; + humanEvidence: boolean; + tasks: Array<{ id: string }>; + }; + expect(tasks.schemaVersion).toBe(1); + expect(tasks.humanEvidence).toBe(false); + expect(tasks.tasks.map((task) => task.id)).toEqual(expectedTaskIds); }); }); diff --git a/scripts/fixtures/human-uat/repos.mjs b/scripts/fixtures/human-uat/repos.mjs new file mode 100644 index 0000000..45e3b3c --- /dev/null +++ b/scripts/fixtures/human-uat/repos.mjs @@ -0,0 +1,327 @@ +/** Deterministic human-UAT fixture trees for #692 kit smoke (not human evidence). */ + +export const HUMAN_UAT_TASK_IDS = [ + "UAT-HUMAN-1", + "UAT-HUMAN-2", + "UAT-HUMAN-3", + "UAT-HUMAN-4", + "UAT-HUMAN-5", + "UAT-HUMAN-6", + "UAT-HUMAN-7", + "UAT-HUMAN-8" +]; + +export function plantedBaselineFiles() { + return { + ".gitignore": [".codedecay/local/", "codedecay-*.json", "codedecay-*.md", ""].join("\n"), + "README.md": [ + "# Acme Billing Lookup", + "", + "Synthetic UAT fixture. Invariant: anonymous `GET /api/invoices` must return `401`.", + "", + "Ambiguous requirement seed (UAT-HUMAN-2):", + "> Make invoice lookup safer for operators.", + "", + "Participants must ask whether anonymous callers may see invoice totals.", + "" + ].join("\n"), + "package.json": + JSON.stringify( + { + name: "codedecay-human-uat-planted", + private: true, + type: "module", + scripts: { + test: "node --test test/unit/session.test.js", + "probe:anonymous": "node scripts/probe-anonymous.mjs" + } + }, + null, + 2 + ) + "\n", + ".codedecay/config.yml": [ + "version: 1", + "commands:", + " test:", + " - npm test", + "probes:", + " - name: anonymous invoice probe", + " command: npm run probe:anonymous", + " timeoutMs: 10000", + "safety:", + " allowCommands: true", + " commandTimeoutMs: 30000", + "llm:", + " provider: disabled", + "" + ].join("\n"), + ".codedecay/memory.json": JSON.stringify( + { + version: 1, + flows: [ + { + name: "Invoice lookup", + areas: ["api", "auth"], + checks: ["anonymous request", "missing authorization header"] + } + ], + invariants: [ + { + name: "Invoices fail closed", + description: "GET /api/invoices must return 401 when Authorization is missing.", + areas: ["api", "auth"], + severity: "high" + } + ] + }, + null, + 2 + ), + "src/auth/session.js": [ + "export function requireSession(authorization) {", + " if (!authorization) return null;", + " return { userId: 'u_42', role: 'ops' };", + "}", + "" + ].join("\n"), + "src/routes/invoices.js": [ + "export function getInvoices(session) {", + " if (!session) return { status: 401, body: { error: 'unauthorized' } };", + " return { status: 200, body: [{ id: 'inv_1', total: 12 }] };", + "}", + "" + ].join("\n"), + "src/server.js": [ + "import { createServer } from 'node:http';", + "import { requireSession } from './auth/session.js';", + "import { getInvoices } from './routes/invoices.js';", + "", + "export function createApp() {", + " return createServer((request, response) => {", + " const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname;", + " if (request.method !== 'GET' || path !== '/api/invoices') {", + " response.writeHead(404).end();", + " return;", + " }", + " const session = requireSession(request.headers.authorization);", + " const result = getInvoices(session);", + " response.writeHead(result.status, { 'content-type': 'application/json' });", + " response.end(JSON.stringify(result.body));", + " });", + "}", + "" + ].join("\n"), + "scripts/probe-anonymous.mjs": [ + "import { createApp } from '../src/server.js';", + "import { requestApp } from './request-app.mjs';", + "", + "const response = await requestApp(createApp(), '/api/invoices');", + "const body = JSON.parse(response.body || '{}');", + "console.log(JSON.stringify({ flow: 'anonymous GET /api/invoices', expected: 401, actual: response.status, body }));", + "if (response.status !== 401) process.exitCode = 1;", + "" + ].join("\n"), + "scripts/request-app.mjs": requestAppHelper(), + "test/unit/session.test.js": [ + "import { test } from 'node:test';", + "import { strictEqual } from 'node:assert/strict';", + "import { requireSession } from '../../src/auth/session.js';", + "", + "test('keeps the authenticated user id', () => {", + " strictEqual(requireSession('Bearer demo-token')?.userId, 'u_42');", + "});", + "" + ].join("\n") + }; +} + +export function plantedRiskyFiles() { + return { + "src/auth/session.js": [ + "export function requireSession(authorization) {", + " // Keep ops dashboards usable when identity is unavailable.", + " if (!authorization) return { userId: 'anonymous', role: 'ops' };", + " return { userId: 'u_42', role: 'ops' };", + "}", + "" + ].join("\n"), + "src/routes/invoices.js": [ + "export function getInvoices(session) {", + " return {", + " status: 200,", + " body: [{ id: 'inv_1', total: 12, role: session?.role ?? 'ops' }]", + " };", + "}", + "" + ].join("\n"), + "test/unit/session.test.js": [ + "import { test } from 'node:test';", + "import { strictEqual } from 'node:assert/strict';", + "import { requireSession } from '../../src/auth/session.js';", + "", + "test('keeps the authenticated user id', () => {", + " strictEqual(requireSession('Bearer demo-token')?.userId, 'u_42');", + "});", + "" + ].join("\n") + }; +} + +/** Clean decoy: docs-only change should not force repair. */ +export function decoyBaselineFiles() { + return { + ".gitignore": [".codedecay/local/", "codedecay-*.json", "codedecay-*.md", ""].join("\n"), + "README.md": "# Acme Docs\n\nClean decoy fixture for UAT-HUMAN-7.\n", + "package.json": + JSON.stringify( + { + name: "codedecay-human-uat-decoy", + private: true, + type: "module", + scripts: { test: "node --test test/ok.test.js" } + }, + null, + 2 + ) + "\n", + ".codedecay/config.yml": [ + "version: 1", + "commands:", + " test:", + " - npm test", + "safety:", + " allowCommands: true", + " commandTimeoutMs: 15000", + "llm:", + " provider: disabled", + "" + ].join("\n"), + "src/ok.js": "export function ok() { return true; }\n", + "test/ok.test.js": [ + "import { test } from 'node:test';", + "import { strictEqual } from 'node:assert/strict';", + "import { ok } from '../src/ok.js';", + "", + "test('ok', () => {", + " strictEqual(ok(), true);", + "});", + "" + ].join("\n") + }; +} + +export function decoyChangedFiles() { + return { + "README.md": "# Acme Docs\n\nClean decoy fixture for UAT-HUMAN-7.\n\nDocs-only clarifying note.\n" + }; +} + +/** Unsafe-target fixture: configured commands present but execution disabled. */ +export function unsafeTargetFiles() { + return { + ".gitignore": [".codedecay/local/", ""].join("\n"), + "README.md": "# Unsafe action fixture (UAT-HUMAN-6)\n", + "package.json": + JSON.stringify( + { + name: "codedecay-human-uat-unsafe", + private: true, + type: "module", + scripts: { + test: "node -e \"console.log('should-not-run')\"", + "danger:rm": "rm -rf /tmp/codedecay-human-uat-should-not-delete" + } + }, + null, + 2 + ) + "\n", + ".codedecay/config.yml": [ + "version: 1", + "commands:", + " test:", + " - npm test", + "safety:", + " allowCommands: false", + " commandTimeoutMs: 5000", + "llm:", + " provider: disabled", + "" + ].join("\n") + }; +} + +function requestAppHelper() { + return [ + "import { Agent, request } from 'node:http';", + "import { Duplex } from 'node:stream';", + "", + "class MemorySocket extends Duplex {", + " #peer;", + " connect(peer) { this.#peer = peer; }", + " _read() {}", + " _write(chunk, _encoding, callback) { this.#peer.push(Buffer.from(chunk)); callback(); }", + " _final(callback) { this.#peer.push(null); callback(); }", + " setNoDelay() { return this; }", + " setKeepAlive() { return this; }", + " setTimeout() { return this; }", + "}", + "", + "function makeSocketPair() {", + " const client = new MemorySocket();", + " const server = new MemorySocket();", + " client.connect(server);", + " server.connect(client);", + " return { client, server };", + "}", + "", + "function makeRequest(options) {", + " return new Promise((resolve, reject) => {", + " const outgoing = request(options, (response) => {", + " let body = '';", + " response.setEncoding('utf8');", + " response.on('data', (chunk) => { body += chunk; });", + " response.on('end', () => resolve({ body, status: response.statusCode }));", + " });", + " outgoing.on('error', reject);", + " outgoing.end();", + " });", + "}", + "", + "export async function requestApp(app, path, { headers } = {}) {", + " try {", + " await new Promise((resolve, reject) => {", + " app.once('error', reject);", + " app.listen(0, '127.0.0.1', resolve);", + " });", + " const address = app.address();", + " return await makeRequest({", + " headers,", + " host: '127.0.0.1',", + " method: 'GET',", + " path,", + " port: address.port", + " });", + " } catch (error) {", + " if (error.code !== 'EPERM') throw error;", + " const sockets = makeSocketPair();", + " const agent = new Agent();", + " agent.createConnection = () => sockets.client;", + " app.emit('connection', sockets.server);", + " return await makeRequest({", + " agent,", + " headers,", + " host: 'in-memory',", + " method: 'GET',", + " path,", + " port: 80", + " });", + " } finally {", + " if (app.listening) {", + " await new Promise((resolve, reject) => {", + " app.close((error) => (error ? reject(error) : resolve()));", + " });", + " }", + " }", + "}", + "" + ].join("\n"); +} diff --git a/scripts/human-uat-setup.mjs b/scripts/human-uat-setup.mjs new file mode 100644 index 0000000..35a17bc --- /dev/null +++ b/scripts/human-uat-setup.mjs @@ -0,0 +1,137 @@ +#!/usr/bin/env node +import { isAbsolute, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { readOptionValue } from "./lib/args.mjs"; +import { resetDir, writeFiles, writeJsonFile } from "./lib/files.mjs"; +import { initFixtureGitRepo, runGit, runGitOutput } from "./lib/git.mjs"; +import { + decoyBaselineFiles, + decoyChangedFiles, + plantedBaselineFiles, + plantedRiskyFiles, + unsafeTargetFiles +} from "./fixtures/human-uat/repos.mjs"; + +const repoRoot = resolve(fileURLToPath(new URL("..", import.meta.url))); +const options = parseArgs(process.argv.slice(2)); + +if (options.help) { + printHelp(); + process.exit(0); +} + +const allowedOutputRoot = resolve(repoRoot, ".codedecay/local/human-uat"); +const outputRoot = resolve(repoRoot, options.outputDir ?? ".codedecay/local/human-uat/fixtures"); +assertSafeOutputDir(outputRoot); + +const plantedDir = resolve(outputRoot, "planted"); +const decoyDir = resolve(outputRoot, "decoy"); +const unsafeDir = resolve(outputRoot, "unsafe"); + +const planted = materializePlanted(plantedDir); +const decoy = materializeDecoy(decoyDir); +const unsafe = materializeUnsafe(unsafeDir); + +const manifest = { + schemaVersion: 1, + purpose: "human-uat-fixture-materialization", + humanEvidence: false, + outputRoot, + planted, + decoy, + unsafe +}; + +writeJsonFile(resolve(outputRoot, "manifest.json"), manifest); + +process.stdout.write( + [ + "Human UAT fixtures ready (not human evidence).", + `Planted: ${plantedDir}`, + ` base=${planted.base} head=${planted.head}`, + `Decoy: ${decoyDir}`, + ` base=${decoy.base} head=${decoy.head}`, + `Unsafe: ${unsafeDir}`, + ` head=${unsafe.head}`, + "" + ].join("\n") +); + +function materializePlanted(dir) { + resetDir(dir); + writeFiles(dir, plantedBaselineFiles()); + const base = initFixtureGitRepo(dir, { + userName: "CodeDecay Human UAT", + commitMessage: "baseline: protect invoice API" + }); + writeFiles(dir, plantedRiskyFiles()); + runGit(dir, ["add", "."]); + runGit(dir, ["commit", "-m", "feat: keep invoice lookup available without auth"]); + const head = runGitOutput(dir, ["rev-parse", "HEAD"]).trim(); + return { dir, base, head, kind: "planted-defect" }; +} + +function materializeDecoy(dir) { + resetDir(dir); + writeFiles(dir, decoyBaselineFiles()); + const base = initFixtureGitRepo(dir, { + userName: "CodeDecay Human UAT", + commitMessage: "baseline: docs decoy" + }); + writeFiles(dir, decoyChangedFiles()); + runGit(dir, ["add", "."]); + runGit(dir, ["commit", "-m", "docs: clarify README"]); + const head = runGitOutput(dir, ["rev-parse", "HEAD"]).trim(); + return { dir, base, head, kind: "clean-decoy" }; +} + +function materializeUnsafe(dir) { + resetDir(dir); + writeFiles(dir, unsafeTargetFiles()); + const head = initFixtureGitRepo(dir, { + userName: "CodeDecay Human UAT", + commitMessage: "baseline: unsafe target with allowCommands false" + }); + return { dir, base: head, head, kind: "unsafe-blocked" }; +} + +function parseArgs(args) { + const normalized = args[0] === "--" ? args.slice(1) : args; + const parsed = { outputDir: undefined, help: false }; + for (let index = 0; index < normalized.length; index += 1) { + const arg = normalized[index]; + if (arg === "--help" || arg === "-h") { + parsed.help = true; + continue; + } + if (arg === "--output-dir") { + parsed.outputDir = readOptionValue(normalized, ++index, arg); + continue; + } + throw new Error(`Unknown option: ${arg}`); + } + return parsed; +} + +function assertSafeOutputDir(candidate) { + const relativePath = relative(allowedOutputRoot, candidate); + if (!relativePath || relativePath.startsWith("..") || isAbsolute(relativePath)) { + throw new Error("The fixture output must be a child of .codedecay/local/human-uat/."); + } +} + +function printHelp() { + process.stdout.write( + [ + "Usage: node scripts/human-uat-setup.mjs [options]", + "", + "Materialize planted, clean-decoy, and unsafe-target fixtures for human UAT.", + "This is kit scaffolding only — not independent human acceptance evidence.", + "", + "Options:", + " --output-dir Destination under .codedecay/local/human-uat/", + " --help Show this help", + "" + ].join("\n") + ); +} diff --git a/scripts/human-uat-smoke.mjs b/scripts/human-uat-smoke.mjs new file mode 100644 index 0000000..f75233a --- /dev/null +++ b/scripts/human-uat-smoke.mjs @@ -0,0 +1,287 @@ +#!/usr/bin/env node +/** + * Deterministic human-UAT workflow smoke for CI drift detection. + * Explicitly NOT independent human acceptance evidence for issue 692. + */ +import { existsSync, mkdirSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createRunId, readOptionValue, splitCommand } from "./lib/args.mjs"; +import { resetDir, writeJsonFile } from "./lib/files.mjs"; +import { runCommand } from "./lib/process.mjs"; +import { HUMAN_UAT_TASK_IDS } from "./fixtures/human-uat/repos.mjs"; + +const repoRoot = resolve(fileURLToPath(new URL("..", import.meta.url))); +const options = parseArgs(process.argv.slice(2)); +const runId = options.runId ?? createRunId(); +const runDir = resolve(repoRoot, options.outputDir ?? `.codedecay/local/human-uat/smoke/${runId}`); +const fixtureRoot = join(runDir, "fixtures"); +const kitRoot = join(repoRoot, "docs/evals/uat-kit"); +const cliCommand = options.cli + ? splitCommand(options.cli) + : ["node", join(repoRoot, "packages/cli/dist/index.js")]; + +if (options.help) { + printHelp(); + process.exit(0); +} + +assertBuiltCli(); +resetDir(runDir); +mkdirSync(fixtureRoot, { recursive: true }); + +const checks = []; +const report = { + schemaVersion: 1, + tool: "CodeDecay human-UAT deterministic smoke", + humanEvidence: false, + purpose: "workflow-drift-detection-only", + startedAt: new Date().toISOString(), + finishedAt: undefined, + status: "running", + runId, + runDir, + cliCommand, + checks: [] +}; + +try { + checkKitFiles(checks); + checkTasksManifest(checks); + + const setup = runCommand(process.execPath, [join(repoRoot, "scripts/human-uat-setup.mjs"), "--output-dir", fixtureRoot], { + cwd: repoRoot, + timeoutMs: 60_000 + }); + record("fixture-setup", setup, checks, (result) => { + if (result.exitCode !== 0) { + throw new Error(`human-uat-setup failed: ${result.stderr || result.stdout}`); + } + }); + + const plantedDir = join(fixtureRoot, "planted"); + const decoyDir = join(fixtureRoot, "decoy"); + const unsafeDir = join(fixtureRoot, "unsafe"); + const manifest = JSON.parse(readFileSync(join(fixtureRoot, "manifest.json"), "utf8")); + + const unit = runCommand("npm", ["test"], { cwd: plantedDir, timeoutMs: 30_000 }); + record("planted-unit-test-passes", unit, checks, (result) => { + if (result.exitCode !== 0) { + throw new Error("Planted fixture unit test must pass (weak happy path)."); + } + }); + + const probe = runCommand("npm", ["run", "probe:anonymous"], { cwd: plantedDir, timeoutMs: 30_000 }); + record("planted-anonymous-probe-fails", probe, checks, (result) => { + if (result.exitCode === 0) { + throw new Error("Planted anonymous probe must fail (expected HTTP != 401)."); + } + }); + + const plantedAnalyzePath = join(runDir, "planted-analyze.json"); + const plantedAnalyze = runCli( + ["analyze", "--cwd", plantedDir, "--base", manifest.planted.base, "--format", "json", "--output", plantedAnalyzePath], + checks, + "planted-analyze" + ); + const plantedReport = JSON.parse(readFileSync(plantedAnalyzePath, "utf8")); + assert(plantedAnalyze.exitCode === 0 || plantedAnalyze.exitCode === 1, "analyze should exit 0/1"); + assert( + Number(plantedReport?.summary?.riskScore ?? 0) > 0 || + (Array.isArray(plantedReport?.findings) && plantedReport.findings.length > 0) || + (Array.isArray(plantedReport?.changedFiles) && plantedReport.changedFiles.length > 0), + "planted analyze must surface risk/findings/changed files" + ); + checks.push({ + id: "UAT-HUMAN-3/4-signal", + ok: true, + detail: `planted riskScore=${plantedReport?.summary?.riskScore ?? "n/a"} findings=${plantedReport?.findings?.length ?? 0}` + }); + + const decoyAnalyzePath = join(runDir, "decoy-analyze.json"); + runCli( + ["analyze", "--cwd", decoyDir, "--base", manifest.decoy.base, "--format", "json", "--output", decoyAnalyzePath], + checks, + "decoy-analyze" + ); + const decoyReport = JSON.parse(readFileSync(decoyAnalyzePath, "utf8")); + const decoyRisk = String(decoyReport?.summary?.riskLevel ?? decoyReport?.summary?.level ?? "").toLowerCase(); + const decoyScore = Number(decoyReport?.summary?.riskScore ?? 0); + assert( + decoyRisk === "low" || decoyScore < Number(plantedReport?.summary?.riskScore ?? 100), + "clean decoy must stay lower risk than planted defect" + ); + checks.push({ + id: "UAT-HUMAN-7-decoy", + ok: true, + detail: `decoy riskLevel=${decoyRisk || "n/a"} riskScore=${decoyScore}` + }); + + const executePath = join(runDir, "unsafe-execute.json"); + const execute = runCli( + ["execute", "--cwd", unsafeDir, "--format", "json", "--output", executePath], + checks, + "unsafe-execute-blocked" + ); + const executeReport = JSON.parse(readFileSync(executePath, "utf8")); + const executeStatus = String(executeReport?.summary?.status ?? "").toLowerCase(); + assert( + executeStatus === "skipped" || execute.exitCode === 0, + "unsafe fixture execute must skip when allowCommands is false" + ); + const skipped = + Number(executeReport?.summary?.skipped ?? 0) > 0 || + (Array.isArray(executeReport?.results) && + executeReport.results.some((row) => String(row.status).toLowerCase() === "skipped")); + assert(skipped, "at least one configured command must be skipped under allowCommands: false"); + checks.push({ + id: "UAT-HUMAN-6-unsafe-block", + ok: true, + detail: `execute status=${executeStatus} skipped=${executeReport?.summary?.skipped ?? 0}` + }); + + const aiHelp = runCli(["ai", "--help"], checks, "ai-help-discoverable"); + assert(/preflight|ai/i.test(aiHelp.stdout), "ai workflow help must be discoverable"); + checks.push({ id: "UAT-HUMAN-1-discover", ok: true, detail: "codedecay ai --help ok" }); + + report.status = "passed"; + report.checks = checks; + report.finishedAt = new Date().toISOString(); + writeJsonFile(join(runDir, "smoke.json"), report); + writeJsonFile(join(runDir, "summary.json"), { + humanEvidence: false, + status: "passed", + note: "Deterministic kit/smoke only. Independent human sessions are still required." + }); + + process.stdout.write( + [ + "Human UAT deterministic smoke passed.", + `Run: ${runDir}`, + "humanEvidence: false", + "" + ].join("\n") + ); +} catch (error) { + report.status = "failed"; + report.checks = checks; + report.error = error instanceof Error ? error.message : String(error); + report.finishedAt = new Date().toISOString(); + writeJsonFile(join(runDir, "smoke.json"), report); + console.error(report.error); + process.exitCode = 1; +} + +function checkKitFiles(checks) { + for (const file of [ + "README.md", + "participant-script.md", + "observer-rubric.md", + "consent-privacy.md", + "result.schema.json", + "summary.template.md", + "facilitator-runbook.md", + "tasks.json", + "fixtures.md" + ]) { + const path = join(kitRoot, file); + assert(existsSync(path), `missing kit file: ${file}`); + } + checks.push({ id: "kit-files", ok: true, detail: "versioned kit files present" }); +} + +function checkTasksManifest(checks) { + const tasks = JSON.parse(readFileSync(join(kitRoot, "tasks.json"), "utf8")); + assert(tasks.schemaVersion === 1, "tasks.json schemaVersion must be 1"); + assert(tasks.humanEvidence === false, "tasks.json must set humanEvidence:false"); + const ids = (tasks.tasks ?? []).map((task) => task.id); + for (const id of HUMAN_UAT_TASK_IDS) { + assert(ids.includes(id), `tasks.json missing ${id}`); + } + checks.push({ id: "tasks-manifest", ok: true, detail: `tasks=${ids.length}` }); +} + +function runCli(args, checks, id) { + const result = runCommand(cliCommand[0], [...cliCommand.slice(1), ...args], { + cwd: repoRoot, + timeoutMs: 120_000 + }); + record(id, result, checks, () => { + /* callers assert semantics */ + }); + return result; +} + +function record(id, result, checks, assertFn) { + const entry = { + id, + ok: result.exitCode === 0 || result.exitCode === 1, + exitCode: result.exitCode, + durationMs: result.durationMs + }; + try { + assertFn(result); + entry.ok = true; + } catch (error) { + entry.ok = false; + entry.error = error instanceof Error ? error.message : String(error); + checks.push(entry); + throw error; + } + checks.push(entry); +} + +function assertBuiltCli() { + const cliPath = join(repoRoot, "packages/cli/dist/index.js"); + if (!options.cli && !existsSync(cliPath)) { + throw new Error("Built CLI missing. Run pnpm build:packages first."); + } +} + +function assert(condition, message) { + if (!condition) throw new Error(message); +} + +function parseArgs(args) { + const normalized = args[0] === "--" ? args.slice(1) : args; + const parsed = { runId: undefined, outputDir: undefined, cli: undefined, help: false }; + for (let index = 0; index < normalized.length; index += 1) { + const arg = normalized[index]; + if (arg === "--help" || arg === "-h") { + parsed.help = true; + continue; + } + if (arg === "--run-id") { + parsed.runId = readOptionValue(normalized, ++index, arg); + continue; + } + if (arg === "--output-dir") { + parsed.outputDir = readOptionValue(normalized, ++index, arg); + continue; + } + if (arg === "--cli") { + parsed.cli = readOptionValue(normalized, ++index, arg); + continue; + } + throw new Error(`Unknown option: ${arg}`); + } + return parsed; +} + +function printHelp() { + process.stdout.write( + [ + "Usage: node scripts/human-uat-smoke.mjs [options]", + "", + "Deterministic CI smoke for human-UAT kit/fixtures drift.", + "Does not count as independent human acceptance evidence.", + "", + "Options:", + " --run-id ", + " --output-dir ", + " --cli ", + " --help", + "" + ].join("\n") + ); +}