diff --git a/CHANGELOG.md b/CHANGELOG.md index 049b805..c0b1f36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) + [Semantic Ver ## [Unreleased] +### Fixed +- **The same input lists its findings in the same order.** The scanners run in + parallel and Trivy does not list its results in a fixed order, so two runs on + the same commit could list the same findings in a different order (on + juice-shop two Trivy findings in one Terraform file swapped), and the JSON + report differed although the README promises byte-identical findings. + Findings are now sorted by scanner, file, line, column, rule and image before + any report is written. Versions up to 0.3.2 have the unstable order. + ## [0.3.2] - 2026-11-15 ### Fixed diff --git a/README.md b/README.md index 9b5d1f6..e7f302e 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,9 @@ One command. One report. One exit code. **Status.** Early release (`v0.3.2`). From `v0.3.0` on, the tag-triggered release workflow publishes to npm with provenance and creates the signed GitHub Release; `v0.2.14` was published to npm without provenance and without a GitHub Release. Report vulnerabilities via [SECURITY.md](SECURITY.md). -**Accuracy contract.** SecGate's aggregation pipeline is deterministic — same inputs produce JSON-byte-identical findings, score, and gate status across every run. Report files embed a timestamp, so they are byte-identical only when the inputs, the scanner versions and the timestamp are the same: set `SECGATE_TIMESTAMP` (ISO 8601, e.g. `2026-01-01T00:00:00Z`) or, if it is unset, `SOURCE_DATE_EPOCH` (Unix seconds). Both pin `timestamp` in the JSON report and the audit log; an unparsable value exits 2. Without them the timestamp is the wall-clock time. Scanner versions and rule databases are not pinned by SecGate. Two test suites lock the contract: +**Accuracy contract.** SecGate's aggregation pipeline is deterministic — same inputs produce JSON-byte-identical findings, score, and gate status across every run. Findings are sorted by scanner, file, line, column and rule, so their order does not depend on which scanner finishes first or on the order a scanner lists its results in. Report files embed a timestamp, so they are byte-identical only when the inputs, the scanner versions and the timestamp are the same: set `SECGATE_TIMESTAMP` (ISO 8601, e.g. `2026-01-01T00:00:00Z`) or, if it is unset, `SOURCE_DATE_EPOCH` (Unix seconds). Both pin `timestamp` in the JSON report and the audit log; an unparsable value exits 2. Without them the timestamp is the wall-clock time. Scanner versions and rule databases are not pinned by SecGate. Two test suites lock the contract: -- `test/determinism.mjs` — JSON byte-equality across reruns, order-stable dedup, override + ignore applied identically, and byte-identical JSON and HTML report files from two CLI runs with `SECGATE_TIMESTAMP` set. +- `test/determinism.mjs` — JSON byte-equality across reruns, order-stable dedup, override + ignore applied identically, byte-identical JSON and HTML report files from two CLI runs with `SECGATE_TIMESTAMP` set, and the same report when the scanners finish in the opposite order and Trivy lists its results in the opposite order. - `test/golden-secgate.mjs` — hand-crafted 11-finding fixture; expected counts, score (`22`), and gate status are inline so any change surfaces in code review. Full pipeline spec lives in [`docs/aggregation.md`](docs/aggregation.md). diff --git a/docs/aggregation.md b/docs/aggregation.md index a8c1c04..9094889 100644 --- a/docs/aggregation.md +++ b/docs/aggregation.md @@ -21,6 +21,8 @@ Every raw finding from every scanner passes through `makeFindingProcessor` in [` A finding that survives stages 1–7 always lands in `findings[]`. A finding never appears twice unless its `(tool, signature, file, image, line)` tuple differs. +When every scanner has finished, `findings[]` is sorted with `compareFindings` ([`lib/scanners.mjs`](../lib/scanners.mjs)): scanner, file, line, column, signature, image, then the whole finding. The scanners run in parallel and Trivy does not list its results in a fixed order, so the same findings would otherwise come out in a different order from run to run. The reports, the SARIF file and a written baseline all use the sorted list. + --- ## Severity tiers (locked) diff --git a/lib/scanners.mjs b/lib/scanners.mjs index 2ee245b..955494e 100644 --- a/lib/scanners.mjs +++ b/lib/scanners.mjs @@ -243,6 +243,42 @@ export function makeFindingProcessor(config, target, findings, suppressions) { }; } +// Scanners pass paths and locations through unchecked, so both helpers order +// any value: text by its string form, a location as missing, then finite +// numbers, then anything else by its JSON text. +function compareText(a, b) { + const x = String(a ?? ""); + const y = String(b ?? ""); + return x < y ? -1 : x > y ? 1 : 0; +} + +function compareLocation(a, b) { + const rank = v => (v == null ? 0 : Number.isFinite(v) ? 1 : 2); + const ra = rank(a); + const rb = rank(b); + if (ra !== rb) return ra - rb; + if (ra === 1) return a - b; + return ra === 2 ? compareText(JSON.stringify(a), JSON.stringify(b)) : 0; +} + +/** + * Total order for the findings list: scanner, file, line, column, rule, image, + * then the whole finding. The scanners run in parallel and Trivy does not list + * its results in a fixed order, so the list is sorted before the report is + * built: the same findings always give the same report. + */ +export function compareFindings(a, b) { + return ( + compareText(a.tool, b.tool) || + compareText(a.file, b.file) || + compareLocation(a.line, b.line) || + compareLocation(a.col, b.col) || + compareText(a.signature, b.signature) || + compareText(a.image, b.image) || + compareText(JSON.stringify(a), JSON.stringify(b)) + ); +} + function applyOverrides(config, severity, signature) { for (const ov of config.severityOverrides) { if (ov && typeof ov.rule === "string" && typeof ov.severity === "string") { diff --git a/secgate.js b/secgate.js index f2cf9f7..41faf8c 100755 --- a/secgate.js +++ b/secgate.js @@ -5,7 +5,7 @@ import path from "path"; import { fileURLToPath } from "url"; import { loadConfig } from "./lib/config.mjs"; -import { makeFindingProcessor } from "./lib/scanners.mjs"; +import { compareFindings, makeFindingProcessor } from "./lib/scanners.mjs"; import { runGitleaks, runSemgrep, @@ -491,6 +491,10 @@ applyResult("osv", osvRes); applyResult("trivy", trivyRes); applyResult("trivyImage", trivyImgRes); +// The scanners finish in any order and Trivy lists its results in no fixed +// order, so sort: the same findings always give the same report. +findings.sort(compareFindings); + /* ──────────────────────────────────────────────────────────────────────────── BASELINE ──────────────────────────────────────────────────────────────────────────── */ diff --git a/test/determinism.mjs b/test/determinism.mjs index d263626..756c3d3 100644 --- a/test/determinism.mjs +++ b/test/determinism.mjs @@ -20,7 +20,7 @@ import { fileURLToPath } from "url"; const libDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "lib"); -const { makeFindingProcessor } = await import(`${libDir}/scanners.mjs`); +const { compareFindings, makeFindingProcessor } = await import(`${libDir}/scanners.mjs`); const { computeScore, computeToolScores } = await import(`${libDir}/score.mjs`); const { summarize, resolveStatus, TOOLS } = await import(`${libDir}/report.mjs`); const { CONFIG_DEFAULTS } = await import(`${libDir}/config.mjs`); @@ -224,6 +224,97 @@ test("determinism: without the variables, timestamp is wall-clock", () => { assert(Math.abs(Date.now() - t) < 300000, "timestamp is current time"); }); fs.rmSync(scan, { recursive: true, force: true }); + +/* ── compareFindings is a total order, also for odd scanner values ── */ +// Scanners pass locations and paths through unchecked, so null, numeric +// strings, words and numbers can meet in one key. Every input order must +// sort to the same list. +function permutations(list) { + if (list.length <= 1) return [list]; + return list.flatMap((x, i) => permutations([...list.slice(0, i), ...list.slice(i + 1)]).map(p => [x, ...p])); +} +test("determinism: compareFindings sorts every input order the same way, whatever the value types", () => { + const base = { tool: "semgrep", type: "code", severity: "MEDIUM", signature: "same-rule", message: "m", file: "index.js", col: 1 }; + const odd = [ + { ...base, line: null }, + { ...base, line: "1" }, + { ...base, line: "unknown" }, + { ...base, line: 2 }, + { ...base, line: null, file: 10 }, + { ...base, line: null, file: "9" } + ]; + const sorted = new Set(permutations(odd).map(p => JSON.stringify([...p].sort(compareFindings)))); + assertEq(sorted.size, 1, "distinct sort results over all 720 input orders"); +}); + +/* ── scanner output order does not change the report (demo rehearsal, 2026-11-15) ── */ +// Only stub Semgrep and Trivy are on PATH (Gitleaks and osv-scanner are off in +// config, npm audit has no package.json). Between the two runs the stubs finish +// in the opposite order and Trivy lists its two results in the opposite order, +// as real Trivy did on juice-shop. +const SEMGREP_OUT = JSON.stringify({ + results: [{ check_id: "js.eval", path: "index.js", start: { line: 1, col: 1 }, end: { line: 1, col: 9 }, extra: { severity: "WARNING", message: "eval" } }], + errors: [] +}); +const misconfig = (id, line) => ({ ID: id, Severity: "MEDIUM", Title: id, CauseMetadata: { StartLine: line, EndLine: line } }); +const trivyOut = list => JSON.stringify({ Results: [{ Target: "main.tf", Misconfigurations: list }] }); +const TRIVY_A = trivyOut([misconfig("AVD-X-0001", 81), misconfig("AVD-X-0002", 74)]); +const TRIVY_B = trivyOut([misconfig("AVD-X-0002", 74), misconfig("AVD-X-0001", 81)]); +function orderStubs() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "secgate-det-bin-")); + fs.symlinkSync(process.execPath, path.join(dir, "node")); + const stubs = { + semgrep: `if [ "$SECGATE_TEST_ORDER" = "b" ]; then /bin/sleep 1; fi\nprintf '%s' '${SEMGREP_OUT}'`, + trivy: [ + 'if [ "$1" = "image" ]; then echo \'{"Results":[]}\'; exit 0; fi', + `if [ "$SECGATE_TEST_ORDER" = "a" ]; then /bin/sleep 1; printf '%s' '${TRIVY_A}'; else printf '%s' '${TRIVY_B}'; fi` + ].join("\n") + }; + for (const [name, script] of Object.entries(stubs)) { + fs.writeFileSync(path.join(dir, name), `#!/bin/sh\nif [ "$1" = "--version" ]; then echo "${name} 1.0.0"; exit 0; fi\n${script}\n`); + fs.chmodSync(path.join(dir, name), 0o755); + } + return dir; +} +test("determinism: scanner completion order and Trivy result order do not change the report", () => { + const binDir = orderStubs(); + const target = fs.mkdtempSync(path.join(os.tmpdir(), "secgate-det-order-")); + fs.writeFileSync(path.join(target, "index.js"), "eval(input);\n"); + fs.writeFileSync(path.join(target, "main.tf"), "resource \"x\" \"y\" {}\n"); + fs.writeFileSync(path.join(target, ".secgate.config.json"), JSON.stringify({ scanners: { gitleaks: false, osv: false } })); + const run = order => { + const out = fs.mkdtempSync(path.join(os.tmpdir(), "secgate-det-out-")); + const env = { ...process.env, PATH: binDir, SECGATE_TIMESTAMP: "2026-01-01T00:00:00Z", SECGATE_TEST_ORDER: order }; + delete env.SOURCE_DATE_EPOCH; + const r = spawnSync(process.execPath, [bin, target, "--output-dir", out], { encoding: "utf-8", cwd: target, env, timeout: 120000 }); + const read = ext => { + const f = fs.readdirSync(out).find(n => n.endsWith(ext)); + return f ? fs.readFileSync(path.join(out, f)) : null; + }; + const res = { status: r.status, stderr: r.stderr, json: read(".json"), html: read(".html") }; + fs.rmSync(out, { recursive: true, force: true }); + return res; + }; + try { + const a = run("a"); + const b = run("b"); + assert(a.json && b.json && a.html && b.html, `report files written (exit ${a.status}): ${a.stderr.slice(0, 200)}`); + const findings = JSON.parse(a.json.toString()).findings; + assertEq(findings.length, 3, "three findings"); + assertEq( + JSON.stringify(findings.map(f => `${f.tool}:${f.line}`)), + JSON.stringify(["semgrep:1", "trivy:74", "trivy:81"]), + "sorted by scanner, file and line" + ); + assertEq(JSON.stringify(JSON.parse(b.json.toString()).findings), JSON.stringify(findings), "findings JSON identical"); + assert(a.json.equals(b.json), "JSON reports are byte-identical"); + assert(a.html.equals(b.html), "HTML reports are byte-identical"); + } finally { + fs.rmSync(binDir, { recursive: true, force: true }); + fs.rmSync(target, { recursive: true, force: true }); + } +}); + console.log("-------------------------"); console.log(`${passed} passed, ${failed} failed`); process.exit(failed === 0 ? 0 : 1); diff --git a/test/schema.mjs b/test/schema.mjs index 09a4dd8..de3e0db 100644 --- a/test/schema.mjs +++ b/test/schema.mjs @@ -107,12 +107,14 @@ test("gitleaks: extracts file + line + CRITICAL severity", () => { const { report } = runWithStubs(d, { gitleaks: readFixture("gitleaks.json") }); const gl = report.findings.filter(f => f.tool === "gitleaks"); assertEq(gl.length, 2, "gitleaks finding count"); - assertEq(gl[0].severity, "CRITICAL", "severity"); - assertEq(gl[0].file, "src/config/aws.js", "file"); - assertEq(gl[0].line, 42, "line"); - assertEq(gl[0].endLine, 42, "endLine"); - assertEq(gl[0].fixableBy, "manual", "fixableBy"); - assert(gl[0].fixable === false, "manual fixes should not be flagged fixable"); + // Findings are sorted by file, so look the fixture's first leak up by its file. + const aws = gl.find(f => f.file === "src/config/aws.js"); + assert(aws, "finding for src/config/aws.js"); + assertEq(aws.severity, "CRITICAL", "severity"); + assertEq(aws.line, 42, "line"); + assertEq(aws.endLine, 42, "endLine"); + assertEq(aws.fixableBy, "manual", "fixableBy"); + assert(aws.fixable === false, "manual fixes should not be flagged fixable"); fs.rmSync(d, { recursive: true, force: true }); });