diff --git a/.tmp_inflate_git.py b/.tmp_inflate_git.py new file mode 100644 index 00000000..72064c63 --- /dev/null +++ b/.tmp_inflate_git.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +"""Inflate a git loose object and print text (for search-only use).""" +import sys, zlib, pathlib + +def main() -> None: + path = pathlib.Path(sys.argv[1]) + data = zlib.decompress(path.read_bytes()) + # Split header / body + nul = data.find(b"\x00") + header = data[:nul].decode("ascii", "replace") + body = data[nul + 1 :] + sys.stdout.write(header + "\n") + sys.stdout.buffer.write(body) + if not body.endswith(b"\n"): + sys.stdout.write("\n") + +if __name__ == "__main__": + main() diff --git a/scripts/run-harness-tests.sh b/scripts/run-harness-tests.sh new file mode 100644 index 00000000..31c04c88 --- /dev/null +++ b/scripts/run-harness-tests.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Parent/agent helper: run quality-harness tests when Shell(npx/npm) is allowlisted. +set -euo pipefail +cd /work/OpenSwarm/worktree/2d671986-25db-40ef-b19c-1cc4196bebd4 + +echo '=== node_modules check ===' +ls -la node_modules 2>&1 | head -3 +if [[ ! -e node_modules/vitest ]]; then + echo 'vitest missing — running npm ci' + npm ci +fi + +echo '=== vitest ===' +npx vitest run \ + src/verify/qualityHarness.test.ts \ + src/cli/reviewAudit.test.ts \ + src/cli/reviewMaxHarness.smoke.test.ts \ + --reporter=verbose 2>&1 | tee /tmp/vitest-harness-out.txt +echo "vitest_exit=${PIPESTATUS[0]}" + +echo '=== tsc filtered ===' +npx tsc --noEmit -p tsconfig.check.json 2>&1 \ + | rg -n "qualityHarness|reviewAudit|reviewMax|cli\.ts" \ + | head -40 \ + | tee /tmp/tsc-harness-out.txt || true diff --git a/scripts/run-harness-vitest.mjs b/scripts/run-harness-vitest.mjs new file mode 100644 index 00000000..002c47d6 --- /dev/null +++ b/scripts/run-harness-vitest.mjs @@ -0,0 +1,31 @@ +#!/usr/bin/env node +// Temporary harness runner — executes vitest from npm cache when local install lacks vitest. +import { spawnSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = join(dirname(fileURLToPath(import.meta.url)), '..'); +const candidates = [ + join(root, 'node_modules', 'vitest', 'vitest.mjs'), + join(root, 'node_modules', 'vitest', 'dist', 'cli.js'), + '/work/.npm-cache/_npx/5aa325d8ffb78db0/node_modules/vitest/vitest.mjs', + '/work/.npm-cache/_npx/69c381f8ad94b576/node_modules/vitest/vitest.mjs', +]; +const vitestEntry = candidates.find((p) => existsSync(p)); +if (!vitestEntry) { + console.error('vitest not found in node_modules or npx cache'); + process.exit(127); +} +const args = [ + vitestEntry, + 'run', + 'src/verify/qualityHarness.test.ts', + 'src/cli/reviewAudit.test.ts', + 'src/cli/reviewMaxHarness.smoke.test.ts', + '--reporter=verbose', +]; +const result = spawnSync(process.execPath, args, { cwd: root, encoding: 'utf8', env: process.env }); +process.stdout.write(result.stdout ?? ''); +process.stderr.write(result.stderr ?? ''); +process.exit(result.status ?? 1); diff --git a/src/cli.ts b/src/cli.ts index 97c5d456..934fa488 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -356,12 +356,14 @@ program .option('--in-place', 'For --max --fix: edit the current working tree instead of an isolated worktree (no branch, no PR)') .option('--fix-rounds ', 'For --max --fix: optional round cap (default: until clean, with a two-hour safety budget)', parsePositiveIntegerOption) .option('--no-security-audit', 'For --max --fix: disable the default CodeQL audit gate') + .option('--harness-only', 'For --max: skip LLM reviewers; run only the deterministic quality harness (static scan + verify commands)') .option('--no-learn', 'For --max: do not record the audit findings into the repo knowledge memory') .action(async (opts: { path?: string; base?: string; issues?: string | boolean; issuesPerArea?: string | boolean; file?: string | boolean; adapter?: string; debug?: boolean; json?: boolean; sarif?: string; readOnly?: boolean; maxTurns?: number; timeout?: number; max?: boolean; concurrency?: number; maxFilesPerArea?: number; yes?: boolean; dryRun?: boolean; out?: string; linear?: boolean; fallback?: string | boolean; fix?: boolean; inPlace?: boolean; fixRounds?: number; learn?: boolean; securityAudit?: boolean; + harnessOnly?: boolean; }) => { try { // --json/--sarif are declared on the shared `review` command but only the @@ -381,6 +383,11 @@ program process.exitCode = 2; return; } + if (opts.harnessOnly && !opts.max) { + console.error('--harness-only requires --max.'); + process.exitCode = 2; + return; + } if (opts.max) { const { runReviewMaxCommand, reviewMaxResultFailed } = await import('./cli/reviewMaxCommand.js'); const result = await runReviewMaxCommand({ @@ -405,6 +412,7 @@ program fixRounds: opts.fixRounds, learn: opts.learn, securityAudit: opts.securityAudit, + harnessOnly: opts.harnessOnly, }); // Exit contract (INT-3100): 2 = the gate did not run at all (no area // reviewed — quota/infra), 1 = it ran and failed. CI reads only this. diff --git a/src/cli/reviewAudit.test.ts b/src/cli/reviewAudit.test.ts index 3ec54ca9..467edda4 100644 --- a/src/cli/reviewAudit.test.ts +++ b/src/cli/reviewAudit.test.ts @@ -10,6 +10,8 @@ import { runMaxReview, mergeFallback, mergeSecurityAuditFindings, + mergeQualityHarnessResult, + QUALITY_HARNESS_AREA, type AuditArea, type AuditAreaResult, type AuditProgress, @@ -217,6 +219,56 @@ describe('mergeSecurityAuditFindings', () => { }); }); +describe('mergeQualityHarnessResult', () => { + it('always injects a harness area so clean scans still gate-ran', () => { + const base: AuditRun = { + results: [], + summary: aggregateAuditResults([]), + }; + const merged = mergeQualityHarnessResult(base, { + status: 'passed', + filesListed: 3, + filesScanned: 3, + findings: [], + commands: [{ name: 'typecheck', kind: 'typecheck', status: 'pass', detail: 'ok' }], + }); + expect(merged.summary.decision).toBe('approve'); + expect(merged.summary.completed).toBe(1); + expect(merged.results[0]?.area.label).toBe(QUALITY_HARNESS_AREA); + const md = formatAuditReport(merged.summary, 'repo', 'ts'); + expect(md).toContain(QUALITY_HARNESS_AREA); + expect(md).toContain('Verdict: APPROVE'); + }); + + it('rejects when static or command findings are errors', () => { + const base: AuditRun = { + results: [{ + area: { label: 'src', dir: 'src', files: ['src/a.ts'] }, + review: { decision: 'approve', feedback: '', issues: [], recommendedActions: [] }, + }], + summary: aggregateAuditResults([{ + area: { label: 'src', dir: 'src', files: ['src/a.ts'] }, + review: { decision: 'approve', feedback: '', issues: [], recommendedActions: [] }, + }]), + }; + const merged = mergeQualityHarnessResult(base, { + status: 'failed', + filesListed: 1, + filesScanned: 1, + findings: [{ + ruleId: 'openswarm/quality-truncated', + level: 'error', + message: 'too large', + filePath: 'src/a.ts', + }], + commands: [], + }); + expect(merged.summary.decision).toBe('reject'); + expect(merged.summary.issues.some((i) => i.includes('openswarm/quality-truncated'))).toBe(true); + expect(formatAuditReport(merged.summary, 'repo', 'ts')).toContain('Quality openswarm/quality-truncated'); + }); +}); + describe('formatAuditReport (INT-2022)', () => { it('renders markdown with verdict, failures, typed follow-ups, and issues', () => { const summary: AuditSummary = { diff --git a/src/cli/reviewAudit.ts b/src/cli/reviewAudit.ts index 000ac043..0a3a4c37 100644 --- a/src/cli/reviewAudit.ts +++ b/src/cli/reviewAudit.ts @@ -19,10 +19,14 @@ import { isInfraError } from '../adapters/errorClassification.js'; import { c, status } from '../support/colors.js'; import { sanitizeTerminalText } from '../tui/sanitize.js'; import type { SecurityFinding } from '../verify/securityAudit.js'; +import type { QualityHarnessResult } from '../verify/qualityHarness.js'; /** Synthetic area label carrying deterministic CodeQL findings into a review run. */ export const SECURITY_AUDIT_AREA = '.openswarm/codeql-security'; +/** Synthetic area label for the deterministic quality harness (static + verify cmds). */ +export const QUALITY_HARNESS_AREA = '.openswarm/quality-harness'; + /** Add deterministic CodeQL findings as a fixable, synthetic review area. */ export function mergeSecurityAuditFindings(run: AuditRun, findings: readonly SecurityFinding[]): AuditRun { const results = run.results.filter((result) => result.area.label !== SECURITY_AUDIT_AREA); @@ -48,6 +52,49 @@ export function mergeSecurityAuditFindings(run: AuditRun, findings: readonly Sec return { ...run, results, summary: aggregateAuditResults(results) }; } +/** + * Fold the deterministic quality harness into the audit run. Always injects an + * area so `--harness-only` still produces a gate-ran verdict and the markdown + * report records coverage even when the scan is clean. + */ +export function mergeQualityHarnessResult(run: AuditRun, harness: QualityHarnessResult): AuditRun { + const results = run.results.filter((result) => result.area.label !== QUALITY_HARNESS_AREA); + const files = [...new Set( + harness.findings.map((finding) => finding.filePath).filter((file): file is string => Boolean(file)), + )].sort(); + const issues = harness.findings.map((finding) => { + const location = finding.filePath + ? `${finding.filePath}${finding.line ? `:${finding.line}` : ''}` + : 'repository'; + return `Quality ${finding.ruleId} (${location}): ${finding.message}`; + }); + const commandLine = harness.commands.length === 0 + ? 'no verify commands discovered' + : harness.commands.map((c) => `${c.name}:${c.status}`).join(', '); + const coverage = `static ${harness.filesScanned}/${harness.filesListed}; commands: ${commandLine}`; + const decision: ReviewResult['decision'] = harness.findings.some((f) => f.level === 'error') + ? 'reject' + : harness.findings.length > 0 + ? 'revise' + : 'approve'; + const feedback = decision === 'approve' + ? `Deterministic quality harness passed (${coverage}).` + : `Deterministic quality harness findings (${coverage}):\n${issues.join('\n')}`; + results.push({ + area: { label: QUALITY_HARNESS_AREA, dir: '.', files }, + review: { + decision, + feedback, + issues, + recommendedActions: harness.findings.map((finding) => ({ + type: finding.ruleId.includes('command') ? 'test' : 'quality', + title: `Address ${finding.ruleId}${finding.filePath ? ` at ${finding.filePath}${finding.line ? `:${finding.line}` : ''}` : ''}`, + })), + }, + }); + return { ...run, results, summary: aggregateAuditResults(results) }; +} + // Source extensions and test patterns mirror src/knowledge/scanner.ts. Kept // local (not imported) because those are unexported module consts; the audit // only needs the stable subset and drift here is low-risk. diff --git a/src/cli/reviewMaxCommand.tsx b/src/cli/reviewMaxCommand.tsx index dc3bdcf5..d7d9b7b7 100644 --- a/src/cli/reviewMaxCommand.tsx +++ b/src/cli/reviewMaxCommand.tsx @@ -23,6 +23,7 @@ import { oneLineError, mergeFallback, mergeSecurityAuditFindings, + mergeQualityHarnessResult, type AuditArea, type AuditRun, type AuditSummary, @@ -56,6 +57,7 @@ import { loadTrustedVerifyPlan, runDeterministicTester } from '../agents/determi import { buildFixRepositoryContext } from './fixPlanning.js'; import { collectFixRuntimePreflightIssues } from './fixPreflight.js'; import { DEFAULT_SECURITY_AUDIT_CONFIG, listTrackedSecurityFiles, runSecurityAudit, type SecurityFinding } from '../verify/securityAudit.js'; +import { runQualityHarness } from '../verify/qualityHarness.js'; /** * Best-effort verify config: `review --max` must still run in a repo with no — @@ -123,6 +125,11 @@ export interface ReviewMaxOptions { learn?: boolean; /** Disable the default-on CodeQL audit gate. */ securityAudit?: boolean; + /** + * Skip LLM area fan-out and run only the deterministic quality harness + * (static scan + isolated verify commands). (M0 / PLATFORM_ROADMAP) + */ + harnessOnly?: boolean; } export interface ReviewMaxCommandResult { @@ -406,6 +413,39 @@ export async function runReviewMaxCommand(opts: ReviewMaxOptions = {}): Promise< const concurrency = positiveIntegerOption(opts.concurrency, 4, '--concurrency'); const maxFilesPerArea = positiveIntegerOption(opts.maxFilesPerArea, 12, '--max-files-per-area'); + // Deterministic-only path: no LLM cost, no area fan-out. Still writes the + // audit report and participates in the same exit-code contract. (M0) + if (opts.harnessOnly) { + if (opts.fix) { + throw new Error('--harness-only cannot be combined with --fix'); + } + const verifyConfig = loadVerifyConfigBestEffort(); + console.log(status.running('Quality harness') + c.dim(' — static scan + isolated verify commands (no LLM)')); + const harness = await runQualityHarness(cwd, { verify: verifyConfig }); + console.log(c.dim( + ` Quality harness: ${harness.status}, scanned ${harness.filesScanned}/${harness.filesListed}, ` + + `${harness.findings.length} finding(s), ${harness.commands.length} command(s).`, + )); + let run: AuditRun = { results: [], summary: aggregateAuditResults([]) }; + run = mergeQualityHarnessResult(run, harness); + console.log(formatAuditSummary(run.summary)); + + const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); + const report = formatAuditReport(run.summary, basename(cwd) || cwd, ts); + const outPath = opts.out ?? join(cwd, '.openswarm', 'audit', `audit-${ts}.md`); + try { + await mkdir(dirname(outPath), { recursive: true }); + await writeFile(outPath, report, 'utf8'); + console.log(`\nReport saved: ${outPath}`); + } catch (e) { + console.warn(`Could not save report: ${e instanceof Error ? e.message : String(e)}`); + } + return { + decision: run.summary.decision, + gateRan: true, + }; + } + let files: string[]; try { files = listSourceFiles(cwd); @@ -764,6 +804,35 @@ export async function runReviewMaxCommand(opts: ReviewMaxOptions = {}): Promise< } } + // Deterministic quality harness (static full-tree + isolated verify commands). + // Runs for every --max so the final verdict and markdown report always carry + // CodeQL-style coverage evidence, not only LLM area notes. (M0 / AGT-3619) + try { + console.log(`\n${status.running('Quality harness')} ${c.dim('static scan + isolated verify commands')}`); + const harness = await runQualityHarness(workCwd, { verify: verifyConfig }); + console.log(c.dim( + ` Quality harness: ${harness.status}, scanned ${harness.filesScanned}/${harness.filesListed}, ` + + `${harness.findings.length} finding(s), ${harness.commands.length} command(s).`, + )); + run = mergeQualityHarnessResult(run, harness); + if (harness.findings.length > 0) { + console.log(formatAuditSummary(run.summary)); + } + } catch (error) { + run = mergeQualityHarnessResult(run, { + status: 'failed', + filesListed: 0, + filesScanned: 0, + findings: [{ + ruleId: 'openswarm/quality-runtime', + level: 'error', + message: `Quality harness aborted: ${error instanceof Error ? error.message : String(error)}`, + }], + commands: [], + }); + console.warn(status.warn(`Quality harness aborted — recorded as an explicit failure.`)); + } + // (3.6) Persist a markdown report so the result isn't lost to the scrollback. // Built here (after --fix) so it reflects the verified post-fix verdicts — // and so the Linear master issue below embeds the same final state. (INT-2022 / INT-2443) diff --git a/src/cli/reviewMaxHarness.smoke.test.ts b/src/cli/reviewMaxHarness.smoke.test.ts new file mode 100644 index 00000000..ec814cb6 --- /dev/null +++ b/src/cli/reviewMaxHarness.smoke.test.ts @@ -0,0 +1,96 @@ +// CLI smoke for `openswarm review --max --harness-only` (AGT-3619 / M0). +import { execFileSync } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, describe, expect, it } from 'vitest'; + +const roots: string[] = []; +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '../..'); +const cliEntry = join(repoRoot, 'src/cli.ts'); + +async function gitRepo(files: Record): Promise { + const root = await mkdtemp(join(tmpdir(), 'openswarm-harness-cli-')); + roots.push(root); + execFileSync('git', ['init'], { cwd: root, stdio: 'ignore' }); + execFileSync('git', ['config', 'user.email', 'harness@example.test'], { cwd: root, stdio: 'ignore' }); + execFileSync('git', ['config', 'user.name', 'Harness'], { cwd: root, stdio: 'ignore' }); + for (const [name, content] of Object.entries(files)) { + const path = join(root, name); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, content); + } + execFileSync('git', ['add', '-A'], { cwd: root, stdio: 'ignore' }); + execFileSync('git', ['commit', '-m', 'init'], { cwd: root, stdio: 'ignore' }); + return root; +} + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +describe('review --max --harness-only CLI smoke', () => { + it('runs the quality harness without LLM reviewers and writes a markdown report', async () => { + const root = await gitRepo({ + 'src/ok.ts': 'export const value = 1;\n', + 'package.json': JSON.stringify({ name: 'fixture', private: true }), + }); + const out = join(root, 'audit-report.md'); + const stdout = execFileSync( + process.execPath, + ['--import', 'tsx', cliEntry, 'review', '--max', '--harness-only', '--yes', '--no-linear', '--path', root, '--out', out], + { + cwd: repoRoot, + encoding: 'utf8', + env: { + ...process.env, + NO_COLOR: '1', + OPENSWARM_DISABLE_TELEMETRY: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + }, + ); + expect(stdout).toMatch(/Quality harness/i); + expect(stdout).toMatch(/Verdict:\s*APPROVE/i); + const report = await readFile(out, 'utf8'); + expect(report).toContain('.openswarm/quality-harness'); + expect(report).toContain('Verdict: APPROVE'); + }, 120_000); + + it('fails closed when a tracked source file cannot be fully scanned', async () => { + const root = await gitRepo({ + 'src/huge.ts': 'x'.repeat(512 * 1024 + 32), + }); + const out = join(root, 'audit-report.md'); + let code = 0; + let combined = ''; + try { + combined = execFileSync( + process.execPath, + ['--import', 'tsx', cliEntry, 'review', '--max', '--harness-only', '--yes', '--no-linear', '--path', root, '--out', out], + { + cwd: repoRoot, + encoding: 'utf8', + env: { + ...process.env, + NO_COLOR: '1', + OPENSWARM_DISABLE_TELEMETRY: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + }, + ); + } catch (error) { + const failure = error as { status?: number; stdout?: string; stderr?: string }; + code = failure.status ?? 1; + combined = `${failure.stdout ?? ''}\n${failure.stderr ?? ''}`; + } + expect(code).toBe(1); + expect(combined).toMatch(/quality-truncated|Verdict:\s*REJECT/i); + const report = await readFile(out, 'utf8'); + expect(report).toContain('openswarm/quality-truncated'); + expect(report).toContain('Verdict: REJECT'); + }, 120_000); +}); diff --git a/src/verify/qualityHarness.test.ts b/src/verify/qualityHarness.test.ts new file mode 100644 index 00000000..8f5c5842 --- /dev/null +++ b/src/verify/qualityHarness.test.ts @@ -0,0 +1,132 @@ +import { mkdir, mkdtemp, symlink, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + runQualityHarness, + scanStaticQuality, + selectQualitySourceFiles, + type QualityCommandResult, +} from './qualityHarness.js'; + +const roots: string[] = []; + +async function fixture(files: Record): Promise { + const root = await mkdtemp(join(tmpdir(), 'openswarm-quality-harness-')); + roots.push(root); + for (const [name, content] of Object.entries(files)) { + const path = join(root, name); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, content); + } + return root; +} + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +describe('selectQualitySourceFiles', () => { + it('keeps tracked source and drops junk / non-source paths', () => { + expect(selectQualitySourceFiles([ + 'src/a.ts', + 'src/a.ts', + 'README.md', + 'node_modules/x.js', + 'dist/out.js', + 'pkg/main.py', + ])).toEqual(['pkg/main.py', 'src/a.ts']); + }); +}); + +describe('scanStaticQuality', () => { + it('scans every listed file and surfaces critical BS as error findings', async () => { + const root = await fixture({ + 'src/clean.ts': 'export const ok = 1;\n', + 'src/bad.ts': 'try { doWork(); } catch {\n}\n', + }); + const { findings, filesScanned } = await scanStaticQuality(root, ['src/clean.ts', 'src/bad.ts']); + expect(filesScanned).toBe(2); + expect(findings.some((f) => f.ruleId === 'openswarm/quality-bs/exception_hiding' && f.filePath === 'src/bad.ts')).toBe(true); + }); + + it('fails closed on oversize (truncation) rather than skipping the file', async () => { + const root = await fixture({ + 'src/huge.ts': Buffer.alloc(512 * 1024 + 8, 0x61), + }); + const { findings, filesScanned } = await scanStaticQuality(root, ['src/huge.ts']); + expect(filesScanned).toBe(0); + expect(findings).toEqual([expect.objectContaining({ + ruleId: 'openswarm/quality-truncated', + level: 'error', + filePath: 'src/huge.ts', + })]); + }); + + it('fails closed on scope escape and unreadable paths', async () => { + const root = await fixture({ 'src/ok.ts': 'export {};\n' }); + const escaped = await scanStaticQuality(root, ['../outside.ts']); + expect(escaped.findings.some((f) => f.ruleId === 'openswarm/quality-scope')).toBe(true); + + const missing = await scanStaticQuality(root, ['src/missing.ts']); + expect(missing.findings.some((f) => f.ruleId === 'openswarm/quality-read' && f.filePath === 'src/missing.ts')).toBe(true); + }); + + it('refuses symlinked source as a non-regular read failure', async () => { + const root = await fixture({ 'src/real.ts': 'export {};\n' }); + await symlink(join(root, 'src/real.ts'), join(root, 'src/link.ts')); + const { findings } = await scanStaticQuality(root, ['src/link.ts']); + expect(findings.some((f) => f.ruleId === 'openswarm/quality-read' && f.filePath === 'src/link.ts')).toBe(true); + }); +}); + +describe('runQualityHarness', () => { + it('combines static findings with isolated command results', async () => { + const root = await fixture({ + 'src/a.ts': 'export const x = 1;\n', + 'package.json': JSON.stringify({ scripts: { typecheck: 'tsc --noEmit' } }), + }); + const executeCommands = async (): Promise => ([ + { name: 'typecheck', kind: 'typecheck', status: 'fail', detail: 'error TS2304' }, + { name: 'test', kind: 'test', status: 'pass', detail: 'ok' }, + ]); + const result = await runQualityHarness(root, { + sourceFiles: ['src/a.ts'], + staticOnly: false, + verify: { enabled: true, blockOnNewFailures: true, maxCommands: 4 }, + executeCommands, + }); + expect(result.filesScanned).toBe(1); + expect(result.commands).toHaveLength(2); + expect(result.findings.some((f) => f.ruleId === 'openswarm/quality-command/typecheck')).toBe(true); + expect(result.status).toBe('failed'); + }); + + it('passes when static scan is clean and commands pass', async () => { + const root = await fixture({ 'src/a.ts': 'export const x = 1;\n' }); + const result = await runQualityHarness(root, { + sourceFiles: ['src/a.ts'], + staticOnly: true, + }); + expect(result).toMatchObject({ + status: 'passed', + filesListed: 1, + filesScanned: 1, + findings: [], + commands: [], + }); + }); + + it('records verify-plan failures as explicit harness errors', async () => { + const root = await fixture({ + 'src/a.ts': 'export {};\n', + '.openswarm/verify.yaml': 'version: 1\ncommands: []\n', + }); + const result = await runQualityHarness(root, { + sourceFiles: ['src/a.ts'], + verify: { enabled: true, blockOnNewFailures: true, maxCommands: 4 }, + }); + expect(result.status).toBe('failed'); + expect(result.findings.some((f) => f.ruleId === 'openswarm/quality-commands')).toBe(true); + }); +}); diff --git a/src/verify/qualityHarness.ts b/src/verify/qualityHarness.ts new file mode 100644 index 00000000..dc05bd2d --- /dev/null +++ b/src/verify/qualityHarness.ts @@ -0,0 +1,410 @@ +// ============================================ +// OpenSwarm - deterministic CodeQL-style quality harness +// ============================================ +// +// Full-tree, fail-closed inspection for `openswarm review --max`: +// 1. Enumerate every tracked source file (no silent skips). +// 2. Read each file with a hard byte ceiling; read / scope / truncation +// failures become explicit error findings — never "passed with gaps". +// 3. Run discover / `.openswarm/verify.yaml` quality commands inside the +// existing isolated verify sandbox. +// +// This is the M0 engine behind hygiene-style inspection (PLATFORM_ROADMAP). + +import { execFile } from 'node:child_process'; +import { constants } from 'node:fs'; +import { access, open } from 'node:fs/promises'; +import { delimiter, extname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { promisify } from 'node:util'; + +import { resolveBaseRef } from '../support/worktreeManager.js'; +import type { VerifyConfig } from '../core/types.js'; +import { scanFileContent } from '../registry/bsDetector.js'; +import { loadTrustedVerifyPlan } from '../agents/deterministicTester.js'; +import type { VerifyCommand } from './manifest.js'; +import { runVerify } from './runner.js'; + +const execFileAsync = promisify(execFile); +const GIT_TIMEOUT_MS = 30_000; +const MAX_SOURCE_BYTES = 512 * 1024; +const OUTPUT_TAIL = 2_000; + +/** Source extensions inspected by the harness (mirrors reviewAudit coverage). */ +const SOURCE_EXTENSIONS = new Set([ + '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', + '.py', '.pyw', + '.rs', '.go', + '.java', '.kt', '.kts', '.scala', '.groovy', + '.c', '.cc', '.cpp', '.cxx', '.h', '.hpp', '.hxx', '.cs', + '.rb', '.php', '.swift', '.m', '.mm', + '.ex', '.exs', '.clj', '.cljs', '.ml', '.mli', '.hs', '.dart', '.lua', '.jl', '.zig', '.nim', +]); + +const SKIP_DIR_SEGMENTS = new Set([ + 'node_modules', 'dist', 'build', 'trash', '.openswarm', 'htmlcov', 'coverage', 'vendor', + 'target', '__pycache__', 'bin', 'obj', +]); + +export type QualityFindingLevel = 'error' | 'warning' | 'note'; + +export interface QualityFinding { + ruleId: string; + level: QualityFindingLevel; + message: string; + filePath?: string; + line?: number; +} + +export interface QualityCommandResult { + name: string; + kind: VerifyCommand['kind']; + status: 'pass' | 'fail' | 'infra' | 'skipped'; + detail: string; +} + +export interface QualityHarnessResult { + status: 'passed' | 'findings' | 'failed'; + filesListed: number; + filesScanned: number; + findings: QualityFinding[]; + commands: QualityCommandResult[]; + detail?: string; +} + +export type QualityCommandExecutor = ( + projectPath: string, + commands: VerifyCommand[], + packageJsonByDirectory: Record, +) => Promise; + +export interface QualityHarnessOptions { + verify?: VerifyConfig; + /** Skip isolated quality commands (static scan only). */ + staticOnly?: boolean; + /** Override the tracked-source listing (tests). */ + sourceFiles?: readonly string[]; + /** Inject isolated command execution (tests). */ + executeCommands?: QualityCommandExecutor; +} + +function shortened(value: string, limit = OUTPUT_TAIL): string { + const flat = value.replace(/\s+/g, ' ').trim(); + if (flat.length <= limit) return flat; + return `${flat.slice(0, limit - 1)}…`; +} + +function inside(root: string, candidate: string): boolean { + const rel = relative(root, candidate); + return rel === '' || (rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel)); +} + +function languageForExtension(ext: string): string | null { + const map: Record = { + '.ts': 'typescript', '.tsx': 'typescript', '.js': 'javascript', '.jsx': 'javascript', + '.mjs': 'javascript', '.cjs': 'javascript', + '.py': 'python', '.pyw': 'python', + '.go': 'go', '.rs': 'rust', '.java': 'java', + '.c': 'c', '.h': 'c', '.cpp': 'cpp', '.cxx': 'cpp', '.cc': 'cpp', + '.hpp': 'cpp', '.hxx': 'cpp', '.cs': 'csharp', + }; + return map[ext] ?? null; +} + +export function selectQualitySourceFiles(paths: readonly string[]): string[] { + return [...new Set(paths.filter((file) => { + if (!file || file.includes('\0')) return false; + const ext = extname(file).toLowerCase(); + if (!SOURCE_EXTENSIONS.has(ext)) return false; + if (file.split(/[/\\]/).some((seg) => SKIP_DIR_SEGMENTS.has(seg))) return false; + return true; + }))].sort(); +} + +async function findGitExecutable(): Promise { + const binary = process.platform === 'win32' ? 'git.exe' : 'git'; + const candidates: string[] = []; + for (const directory of (process.env.PATH ?? '').split(delimiter)) { + if (!isAbsolute(directory)) continue; + candidates.push(join(directory, binary)); + } + const seen = new Set(); + for (const candidate of candidates) { + if (seen.has(candidate)) continue; + seen.add(candidate); + try { + await access(candidate, constants.X_OK); + return candidate; + } catch { + // keep looking + } + } + return undefined; +} + +/** + * Every tracked source path (git index). Missing coverage is an explicit failure — + * the harness must not silently audit a subset. + */ +export async function listTrackedQualitySourceFiles(projectPath: string): Promise { + const git = await findGitExecutable(); + if (!git) throw new Error('git is not available on an absolute PATH entry.'); + try { + const { stdout } = await execFileAsync(git, ['ls-files', '-z'], { + cwd: projectPath, + timeout: GIT_TIMEOUT_MS, + maxBuffer: 64 * 1024 * 1024, + windowsHide: true, + }); + return selectQualitySourceFiles(stdout.split('\u0000')); + } catch (error) { + throw new Error('Could not enumerate tracked source for the quality harness.', { cause: error }); + } +} + +async function readBoundedSource(absolutePath: string): Promise { + const handle = await open(absolutePath, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const info = await handle.stat(); + if (!info.isFile() || info.isSymbolicLink()) { + throw new Error('source must be a regular file'); + } + if (info.size > MAX_SOURCE_BYTES) { + const err = new Error(`source exceeds ${MAX_SOURCE_BYTES} bytes`); + (err as NodeJS.ErrnoException & { code?: string }).code = 'QUALITY_TRUNCATED'; + throw err; + } + const buffer = Buffer.alloc(MAX_SOURCE_BYTES + 1); + let offset = 0; + while (offset < buffer.length) { + const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, null); + if (bytesRead === 0) break; + offset += bytesRead; + } + if (offset > MAX_SOURCE_BYTES) { + const err = new Error(`source exceeds ${MAX_SOURCE_BYTES} bytes`); + (err as NodeJS.ErrnoException & { code?: string }).code = 'QUALITY_TRUNCATED'; + throw err; + } + const bytes = buffer.subarray(0, offset); + if (bytes.includes(0)) { + const err = new Error('source contains NUL bytes'); + (err as NodeJS.ErrnoException & { code?: string }).code = 'QUALITY_BINARY'; + throw err; + } + return bytes.toString('utf8'); + } finally { + await handle.close(); + } +} + +/** + * Static pass over every listed path. A path that cannot be fully read yields an + * error finding so the harness cannot report a clean scan with holes. + */ +export async function scanStaticQuality( + projectPath: string, + sourceFiles: readonly string[], +): Promise<{ findings: QualityFinding[]; filesScanned: number }> { + const root = resolve(projectPath); + const findings: QualityFinding[] = []; + let filesScanned = 0; + + for (const file of sourceFiles) { + if (!file || file.includes('\0') || isAbsolute(file) || /(^|\/)\.\.(\/|$)/.test(file)) { + findings.push({ + ruleId: 'openswarm/quality-scope', + level: 'error', + message: `Source path escapes or is invalid for the quality harness: ${file || ''}`, + filePath: file || undefined, + }); + continue; + } + const absolute = resolve(root, file); + if (!inside(root, absolute)) { + findings.push({ + ruleId: 'openswarm/quality-scope', + level: 'error', + message: `Source path escapes repository root: ${file}`, + filePath: file, + }); + continue; + } + + try { + const content = await readBoundedSource(absolute); + filesScanned += 1; + const language = languageForExtension(extname(file).toLowerCase()); + if (!language) continue; + for (const issue of scanFileContent(content, file, language)) { + if (issue.severity === 'minor') continue; + findings.push({ + ruleId: `openswarm/quality-bs/${issue.category}`, + level: issue.severity === 'critical' ? 'error' : 'warning', + message: issue.message, + filePath: file, + line: issue.line > 0 ? issue.line : undefined, + }); + } + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + const message = error instanceof Error ? error.message : String(error); + if (code === 'QUALITY_TRUNCATED') { + findings.push({ + ruleId: 'openswarm/quality-truncated', + level: 'error', + message: `Read truncated — file exceeds the ${MAX_SOURCE_BYTES}-byte quality harness ceiling.`, + filePath: file, + }); + } else if (code === 'QUALITY_BINARY') { + findings.push({ + ruleId: 'openswarm/quality-binary', + level: 'error', + message: 'Source read failed — file contains NUL bytes and cannot be scanned as text.', + filePath: file, + }); + } else { + findings.push({ + ruleId: 'openswarm/quality-read', + level: 'error', + message: `Source read failed — scan coverage is incomplete: ${shortened(message, 240)}`, + filePath: file, + }); + } + } + } + + if (sourceFiles.length > 0 && filesScanned === 0 && findings.every((f) => f.ruleId !== 'openswarm/quality-read')) { + // Enumeration produced paths but none were readable without an explicit finding — + // treat as coverage failure so we never claim a vacuous pass. + const hasCoverageFinding = findings.some((f) => + f.ruleId === 'openswarm/quality-truncated' + || f.ruleId === 'openswarm/quality-binary' + || f.ruleId === 'openswarm/quality-scope' + || f.ruleId === 'openswarm/quality-read'); + if (!hasCoverageFinding) { + findings.push({ + ruleId: 'openswarm/quality-coverage', + level: 'error', + message: `Listed ${sourceFiles.length} source file(s) but scanned none.`, + }); + } + } + + return { findings, filesScanned }; +} + +export async function defaultExecuteQualityCommands( + projectPath: string, + commands: VerifyCommand[], + packageJsonByDirectory: Record, +): Promise { + if (commands.length === 0) return []; + const base = await resolveBaseRef(projectPath).catch((error) => { + throw new Error(`quality-harness: failed to resolve base ref: ${error instanceof Error ? error.message : String(error)}`); + }); + const evidence = await runVerify({ + projectPath, + commands, + baseRef: base.ref, + trustedPackageJsonByDirectory: packageJsonByDirectory, + }); + return evidence.map((item) => { + let status: QualityCommandResult['status']; + if (item.securityFailure) status = 'fail'; + else if (item.headStatus === 'pass') status = 'pass'; + else if (item.headStatus === 'infra') status = 'infra'; + else status = 'fail'; + return { + name: item.command.name, + kind: item.command.kind, + status, + detail: shortened(item.rawOutputTail), + }; + }); +} + +function commandFindings(commands: readonly QualityCommandResult[]): QualityFinding[] { + const findings: QualityFinding[] = []; + for (const command of commands) { + if (command.status === 'pass' || command.status === 'skipped') continue; + findings.push({ + ruleId: `openswarm/quality-command/${command.kind}`, + level: 'error', + message: command.status === 'infra' + ? `Quality command "${command.name}" hit an infrastructure failure: ${command.detail || 'no detail'}` + : `Quality command "${command.name}" failed in isolation: ${command.detail || 'non-zero exit'}`, + }); + } + return findings; +} + +function harnessStatus(findings: readonly QualityFinding[]): QualityHarnessResult['status'] { + if (findings.some((f) => f.level === 'error')) return 'failed'; + if (findings.length > 0) return 'findings'; + return 'passed'; +} + +/** + * Deterministic quality harness: complete static coverage of tracked source, + * then isolated typecheck/lint/test/build commands from verify discovery. + */ +export async function runQualityHarness( + projectPath: string, + options: QualityHarnessOptions = {}, +): Promise { + let filesListed = 0; + let sourceFiles: string[]; + try { + sourceFiles = options.sourceFiles + ? selectQualitySourceFiles(options.sourceFiles) + : await listTrackedQualitySourceFiles(projectPath); + filesListed = sourceFiles.length; + } catch (error) { + const cause = shortened(error instanceof Error ? error.message : String(error)); + return { + status: 'failed', + filesListed: 0, + filesScanned: 0, + findings: [{ + ruleId: 'openswarm/quality-enumerate', + level: 'error', + message: `Could not list tracked source for the quality harness: ${cause}`, + }], + commands: [], + detail: cause, + }; + } + + const staticScan = await scanStaticQuality(projectPath, sourceFiles); + const findings = [...staticScan.findings]; + const commands: QualityCommandResult[] = []; + + const verify = options.verify ?? { enabled: true, blockOnNewFailures: true, maxCommands: 4 }; + if (!options.staticOnly && verify.enabled) { + try { + const plan = await loadTrustedVerifyPlan(projectPath, verify); + const execute = options.executeCommands ?? defaultExecuteQualityCommands; + const results = await execute(projectPath, plan.commands, plan.packageJsonByDirectory); + commands.push(...results); + findings.push(...commandFindings(results)); + } catch (error) { + const cause = shortened(error instanceof Error ? error.message : String(error)); + findings.push({ + ruleId: 'openswarm/quality-commands', + level: 'error', + message: `Quality command planning/execution failed: ${cause}`, + }); + } + } + + return { + status: harnessStatus(findings), + filesListed, + filesScanned: staticScan.filesScanned, + findings, + commands, + ...(filesListed !== staticScan.filesScanned + ? { detail: `Scanned ${staticScan.filesScanned}/${filesListed} tracked source file(s).` } + : {}), + }; +}