Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .tmp_inflate_git.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/usr/bin/env python3
"""Inflate a git loose object and print text (for search-only use)."""
import sys, zlib, pathlib

def main() -> None:
path = pathlib.Path(sys.argv[1])
data = zlib.decompress(path.read_bytes())
# Split header / body
nul = data.find(b"\x00")
header = data[:nul].decode("ascii", "replace")
body = data[nul + 1 :]
sys.stdout.write(header + "\n")
sys.stdout.buffer.write(body)
if not body.endswith(b"\n"):
sys.stdout.write("\n")

if __name__ == "__main__":
main()
25 changes: 25 additions & 0 deletions scripts/run-harness-tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Parent/agent helper: run quality-harness tests when Shell(npx/npm) is allowlisted.
set -euo pipefail
cd /work/OpenSwarm/worktree/2d671986-25db-40ef-b19c-1cc4196bebd4

echo '=== node_modules check ==='
ls -la node_modules 2>&1 | head -3
if [[ ! -e node_modules/vitest ]]; then
echo 'vitest missing — running npm ci'
npm ci
fi

echo '=== vitest ==='
npx vitest run \
src/verify/qualityHarness.test.ts \
src/cli/reviewAudit.test.ts \
src/cli/reviewMaxHarness.smoke.test.ts \
--reporter=verbose 2>&1 | tee /tmp/vitest-harness-out.txt
echo "vitest_exit=${PIPESTATUS[0]}"

echo '=== tsc filtered ==='
npx tsc --noEmit -p tsconfig.check.json 2>&1 \
| rg -n "qualityHarness|reviewAudit|reviewMax|cli\.ts" \
| head -40 \
| tee /tmp/tsc-harness-out.txt || true
31 changes: 31 additions & 0 deletions scripts/run-harness-vitest.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/usr/bin/env node
// Temporary harness runner — executes vitest from npm cache when local install lacks vitest.
import { spawnSync } from 'node:child_process';
import { existsSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';

const root = join(dirname(fileURLToPath(import.meta.url)), '..');
const candidates = [
join(root, 'node_modules', 'vitest', 'vitest.mjs'),
join(root, 'node_modules', 'vitest', 'dist', 'cli.js'),
'/work/.npm-cache/_npx/5aa325d8ffb78db0/node_modules/vitest/vitest.mjs',
'/work/.npm-cache/_npx/69c381f8ad94b576/node_modules/vitest/vitest.mjs',
];
const vitestEntry = candidates.find((p) => existsSync(p));
if (!vitestEntry) {
console.error('vitest not found in node_modules or npx cache');
process.exit(127);
}
const args = [
vitestEntry,
'run',
'src/verify/qualityHarness.test.ts',
'src/cli/reviewAudit.test.ts',
'src/cli/reviewMaxHarness.smoke.test.ts',
'--reporter=verbose',
];
const result = spawnSync(process.execPath, args, { cwd: root, encoding: 'utf8', env: process.env });
process.stdout.write(result.stdout ?? '');
process.stderr.write(result.stderr ?? '');
process.exit(result.status ?? 1);
8 changes: 8 additions & 0 deletions src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -356,12 +356,14 @@ program
.option('--in-place', 'For --max --fix: edit the current working tree instead of an isolated worktree (no branch, no PR)')
.option('--fix-rounds <n>', 'For --max --fix: optional round cap (default: until clean, with a two-hour safety budget)', parsePositiveIntegerOption)
.option('--no-security-audit', 'For --max --fix: disable the default CodeQL audit gate')
.option('--harness-only', 'For --max: skip LLM reviewers; run only the deterministic quality harness (static scan + verify commands)')
.option('--no-learn', 'For --max: do not record the audit findings into the repo knowledge memory')
.action(async (opts: {
path?: string; base?: string; issues?: string | boolean; issuesPerArea?: string | boolean; file?: string | boolean; adapter?: string; debug?: boolean; json?: boolean; sarif?: string; readOnly?: boolean;
maxTurns?: number; timeout?: number;
max?: boolean; concurrency?: number; maxFilesPerArea?: number; yes?: boolean; dryRun?: boolean;
out?: string; linear?: boolean; fallback?: string | boolean; fix?: boolean; inPlace?: boolean; fixRounds?: number; learn?: boolean; securityAudit?: boolean;
harnessOnly?: boolean;
}) => {
try {
// --json/--sarif are declared on the shared `review` command but only the
Expand All @@ -381,6 +383,11 @@ program
process.exitCode = 2;
return;
}
if (opts.harnessOnly && !opts.max) {
console.error('--harness-only requires --max.');
process.exitCode = 2;
return;
}
if (opts.max) {
const { runReviewMaxCommand, reviewMaxResultFailed } = await import('./cli/reviewMaxCommand.js');
const result = await runReviewMaxCommand({
Expand All @@ -405,6 +412,7 @@ program
fixRounds: opts.fixRounds,
learn: opts.learn,
securityAudit: opts.securityAudit,
harnessOnly: opts.harnessOnly,
});
// Exit contract (INT-3100): 2 = the gate did not run at all (no area
// reviewed — quota/infra), 1 = it ran and failed. CI reads only this.
Expand Down
52 changes: 52 additions & 0 deletions src/cli/reviewAudit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import {
runMaxReview,
mergeFallback,
mergeSecurityAuditFindings,
mergeQualityHarnessResult,
QUALITY_HARNESS_AREA,
type AuditArea,
type AuditAreaResult,
type AuditProgress,
Expand Down Expand Up @@ -217,6 +219,56 @@ describe('mergeSecurityAuditFindings', () => {
});
});

describe('mergeQualityHarnessResult', () => {
it('always injects a harness area so clean scans still gate-ran', () => {
const base: AuditRun = {
results: [],
summary: aggregateAuditResults([]),
};
const merged = mergeQualityHarnessResult(base, {
status: 'passed',
filesListed: 3,
filesScanned: 3,
findings: [],
commands: [{ name: 'typecheck', kind: 'typecheck', status: 'pass', detail: 'ok' }],
});
expect(merged.summary.decision).toBe('approve');
expect(merged.summary.completed).toBe(1);
expect(merged.results[0]?.area.label).toBe(QUALITY_HARNESS_AREA);
const md = formatAuditReport(merged.summary, 'repo', 'ts');
expect(md).toContain(QUALITY_HARNESS_AREA);
expect(md).toContain('Verdict: APPROVE');
});

it('rejects when static or command findings are errors', () => {
const base: AuditRun = {
results: [{
area: { label: 'src', dir: 'src', files: ['src/a.ts'] },
review: { decision: 'approve', feedback: '', issues: [], recommendedActions: [] },
}],
summary: aggregateAuditResults([{
area: { label: 'src', dir: 'src', files: ['src/a.ts'] },
review: { decision: 'approve', feedback: '', issues: [], recommendedActions: [] },
}]),
};
const merged = mergeQualityHarnessResult(base, {
status: 'failed',
filesListed: 1,
filesScanned: 1,
findings: [{
ruleId: 'openswarm/quality-truncated',
level: 'error',
message: 'too large',
filePath: 'src/a.ts',
}],
commands: [],
});
expect(merged.summary.decision).toBe('reject');
expect(merged.summary.issues.some((i) => i.includes('openswarm/quality-truncated'))).toBe(true);
expect(formatAuditReport(merged.summary, 'repo', 'ts')).toContain('Quality openswarm/quality-truncated');
});
});

describe('formatAuditReport (INT-2022)', () => {
it('renders markdown with verdict, failures, typed follow-ups, and issues', () => {
const summary: AuditSummary = {
Expand Down
47 changes: 47 additions & 0 deletions src/cli/reviewAudit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,14 @@ import { isInfraError } from '../adapters/errorClassification.js';
import { c, status } from '../support/colors.js';
import { sanitizeTerminalText } from '../tui/sanitize.js';
import type { SecurityFinding } from '../verify/securityAudit.js';
import type { QualityHarnessResult } from '../verify/qualityHarness.js';

/** Synthetic area label carrying deterministic CodeQL findings into a review run. */
export const SECURITY_AUDIT_AREA = '.openswarm/codeql-security';

/** Synthetic area label for the deterministic quality harness (static + verify cmds). */
export const QUALITY_HARNESS_AREA = '.openswarm/quality-harness';

/** Add deterministic CodeQL findings as a fixable, synthetic review area. */
export function mergeSecurityAuditFindings(run: AuditRun, findings: readonly SecurityFinding[]): AuditRun {
const results = run.results.filter((result) => result.area.label !== SECURITY_AUDIT_AREA);
Expand All @@ -48,6 +52,49 @@ export function mergeSecurityAuditFindings(run: AuditRun, findings: readonly Sec
return { ...run, results, summary: aggregateAuditResults(results) };
}

/**
* Fold the deterministic quality harness into the audit run. Always injects an
* area so `--harness-only` still produces a gate-ran verdict and the markdown
* report records coverage even when the scan is clean.
*/
export function mergeQualityHarnessResult(run: AuditRun, harness: QualityHarnessResult): AuditRun {
const results = run.results.filter((result) => result.area.label !== QUALITY_HARNESS_AREA);
const files = [...new Set(
harness.findings.map((finding) => finding.filePath).filter((file): file is string => Boolean(file)),
)].sort();
const issues = harness.findings.map((finding) => {
const location = finding.filePath
? `${finding.filePath}${finding.line ? `:${finding.line}` : ''}`
: 'repository';
return `Quality ${finding.ruleId} (${location}): ${finding.message}`;
});
const commandLine = harness.commands.length === 0
? 'no verify commands discovered'
: harness.commands.map((c) => `${c.name}:${c.status}`).join(', ');
const coverage = `static ${harness.filesScanned}/${harness.filesListed}; commands: ${commandLine}`;
const decision: ReviewResult['decision'] = harness.findings.some((f) => f.level === 'error')
? 'reject'
: harness.findings.length > 0
? 'revise'
: 'approve';
const feedback = decision === 'approve'
? `Deterministic quality harness passed (${coverage}).`
: `Deterministic quality harness findings (${coverage}):\n${issues.join('\n')}`;
results.push({
area: { label: QUALITY_HARNESS_AREA, dir: '.', files },
review: {
decision,
feedback,
issues,
recommendedActions: harness.findings.map((finding) => ({
type: finding.ruleId.includes('command') ? 'test' : 'quality',
title: `Address ${finding.ruleId}${finding.filePath ? ` at ${finding.filePath}${finding.line ? `:${finding.line}` : ''}` : ''}`,
})),
},
});
return { ...run, results, summary: aggregateAuditResults(results) };
}

// Source extensions and test patterns mirror src/knowledge/scanner.ts. Kept
// local (not imported) because those are unexported module consts; the audit
// only needs the stable subset and drift here is low-risk.
Expand Down
69 changes: 69 additions & 0 deletions src/cli/reviewMaxCommand.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import {
oneLineError,
mergeFallback,
mergeSecurityAuditFindings,
mergeQualityHarnessResult,
type AuditArea,
type AuditRun,
type AuditSummary,
Expand Down Expand Up @@ -56,6 +57,7 @@ import { loadTrustedVerifyPlan, runDeterministicTester } from '../agents/determi
import { buildFixRepositoryContext } from './fixPlanning.js';
import { collectFixRuntimePreflightIssues } from './fixPreflight.js';
import { DEFAULT_SECURITY_AUDIT_CONFIG, listTrackedSecurityFiles, runSecurityAudit, type SecurityFinding } from '../verify/securityAudit.js';
import { runQualityHarness } from '../verify/qualityHarness.js';

/**
* Best-effort verify config: `review --max` must still run in a repo with no —
Expand Down Expand Up @@ -123,6 +125,11 @@ export interface ReviewMaxOptions {
learn?: boolean;
/** Disable the default-on CodeQL audit gate. */
securityAudit?: boolean;
/**
* Skip LLM area fan-out and run only the deterministic quality harness
* (static scan + isolated verify commands). (M0 / PLATFORM_ROADMAP)
*/
harnessOnly?: boolean;
}

export interface ReviewMaxCommandResult {
Expand Down Expand Up @@ -406,6 +413,39 @@ export async function runReviewMaxCommand(opts: ReviewMaxOptions = {}): Promise<
const concurrency = positiveIntegerOption(opts.concurrency, 4, '--concurrency');
const maxFilesPerArea = positiveIntegerOption(opts.maxFilesPerArea, 12, '--max-files-per-area');

// Deterministic-only path: no LLM cost, no area fan-out. Still writes the
// audit report and participates in the same exit-code contract. (M0)
if (opts.harnessOnly) {
if (opts.fix) {
throw new Error('--harness-only cannot be combined with --fix');
}
const verifyConfig = loadVerifyConfigBestEffort();
console.log(status.running('Quality harness') + c.dim(' — static scan + isolated verify commands (no LLM)'));
const harness = await runQualityHarness(cwd, { verify: verifyConfig });
console.log(c.dim(
` Quality harness: ${harness.status}, scanned ${harness.filesScanned}/${harness.filesListed}, `
+ `${harness.findings.length} finding(s), ${harness.commands.length} command(s).`,
));
let run: AuditRun = { results: [], summary: aggregateAuditResults([]) };
run = mergeQualityHarnessResult(run, harness);
console.log(formatAuditSummary(run.summary));

const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
const report = formatAuditReport(run.summary, basename(cwd) || cwd, ts);
const outPath = opts.out ?? join(cwd, '.openswarm', 'audit', `audit-${ts}.md`);
try {
await mkdir(dirname(outPath), { recursive: true });
await writeFile(outPath, report, 'utf8');
console.log(`\nReport saved: ${outPath}`);
} catch (e) {
console.warn(`Could not save report: ${e instanceof Error ? e.message : String(e)}`);
}
return {
decision: run.summary.decision,
gateRan: true,
};
}

let files: string[];
try {
files = listSourceFiles(cwd);
Expand Down Expand Up @@ -764,6 +804,35 @@ export async function runReviewMaxCommand(opts: ReviewMaxOptions = {}): Promise<
}
}

// Deterministic quality harness (static full-tree + isolated verify commands).
// Runs for every --max so the final verdict and markdown report always carry
// CodeQL-style coverage evidence, not only LLM area notes. (M0 / AGT-3619)
try {
console.log(`\n${status.running('Quality harness')} ${c.dim('static scan + isolated verify commands')}`);
const harness = await runQualityHarness(workCwd, { verify: verifyConfig });
console.log(c.dim(
` Quality harness: ${harness.status}, scanned ${harness.filesScanned}/${harness.filesListed}, `
+ `${harness.findings.length} finding(s), ${harness.commands.length} command(s).`,
));
run = mergeQualityHarnessResult(run, harness);
if (harness.findings.length > 0) {
console.log(formatAuditSummary(run.summary));
}
} catch (error) {
run = mergeQualityHarnessResult(run, {
status: 'failed',
filesListed: 0,
filesScanned: 0,
findings: [{
ruleId: 'openswarm/quality-runtime',
level: 'error',
message: `Quality harness aborted: ${error instanceof Error ? error.message : String(error)}`,
}],
commands: [],
});
console.warn(status.warn(`Quality harness aborted — recorded as an explicit failure.`));
}

// (3.6) Persist a markdown report so the result isn't lost to the scrollback.
// Built here (after --fix) so it reflects the verified post-fix verdicts —
// and so the Linear master issue below embeds the same final state. (INT-2022 / INT-2443)
Expand Down
Loading
Loading