From ae86fa0f2a54788297bb2958acd974640eeef048 Mon Sep 17 00:00:00 2001 From: Sergio Duran <14999651+durang@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:13:30 -0700 Subject: [PATCH] Add gateReview to measure a review like ci review --fail-on --- README.md | 22 ++++ src/index.ts | 86 +++++++++++++++ tests/gate.test.mjs | 254 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 362 insertions(+) create mode 100644 tests/gate.test.mjs diff --git a/README.md b/README.md index 84bdeba..bf0b213 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,28 @@ try { } ``` +## Failing a job on open findings + +`gateReview(outcome, { failOn })` measures a `ReviewOutcome` against a `WARNING` or `SERIOUS` threshold with the rules of `proofline ci review --fail-on`, so a Node pipeline fails the same way the CLI does. + +```typescript +import { gateReview } from "@proofline/sdk"; + +const outcome = await client.waitForReview(organizationId, reviewId); +const result = gateReview(outcome, { failOn: "SERIOUS" }); +if (result !== "NOT_REACHED") { + console.error(`Proofline: ${result}, ${outcome.reviewUrl}`); + process.exitCode = { REACHED: 3, NOT_COMPLETE: 2, UNAVAILABLE: 4 }[result]; +} +``` + +- `REACHED`: the completed review has an open finding at or above the threshold. +- `NOT_REACHED`: the completed review has none. +- `NOT_COMPLETE`: the review is incomplete, failed, or stopped, whatever it found. +- `UNAVAILABLE`: the wait timed out, or the completed review lacks valid open-finding counts, so the threshold cannot be measured. + +The server counts open findings from their current decisions, so dismissed findings and unexpired risk acceptances do not count. A code-only finding counts while it is not `RESOLVED` and its complete CODE assessment is of the review's own commits. A severity the SDK does not know reaches every threshold. The exit codes above are the CLI's. The threshold is the caller's and applies to any review; it is separate from the server's opt-in `merge_gate` decision. + ## Coverage uploads Upload CI coverage reports with a repository-restricted token that holds `coverage:write`. The token holder must have current GitHub write access to that repository. Set `PROOFLINE_TOKEN` to that token and pass the full commit SHA that the tests ran on. The server refuses branch names and abbreviated SHAs. diff --git a/src/index.ts b/src/index.ts index f8a73b0..19b36c1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -64,6 +64,23 @@ export interface ReviewOutcome { reason: string | null; } +/** A severity threshold, as `proofline ci review --fail-on` takes it. */ +export type SeverityThreshold = "WARNING" | "SERIOUS"; + +/** + * How a review measures against a threshold. The CLI exits with 3 for + * `REACHED`, 0 for `NOT_REACHED`, 2 for `NOT_COMPLETE` and 4 for `UNAVAILABLE`. + */ +export type GateResult = + | "REACHED" + | "NOT_REACHED" + | "NOT_COMPLETE" + | "UNAVAILABLE"; + +export interface GateOptions { + failOn: SeverityThreshold; +} + export interface WaitOptions { timeoutMs?: number; } @@ -375,6 +392,11 @@ function codeAssessment(review: Review): CodeAssessment | null { // A CODE result cannot replace a missing ENV result or its check receipts. if (review.assessment_count !== 0 || review.assessments.length !== 0) return null; + return recordedCode(review); +} + +/** The CODE record of this review's own commits with published findings. */ +function recordedCode(review: Review): CodeAssessment | null { const code = review.code_assessment; if ( !isObject(code) || @@ -522,3 +544,67 @@ function outcome( } return { status, reviewUrl, review, gaps, reason }; } + +// Least severe first. +const SEVERITIES: readonly string[] = ["WARNING", "SERIOUS"]; + +/** + * Measures a review against a severity threshold with the rules of + * `proofline ci review --fail-on`. + * + * Completeness decides first: an incomplete, failed or stopped review is + * `NOT_COMPLETE` whatever it found, and a wait that timed out is + * `UNAVAILABLE`. A completed review is `REACHED` when the server counts an open + * finding at or above the threshold, or when a complete CODE assessment of the + * review's own commits holds an unresolved finding at or above it. Dismissed + * findings and unexpired risk acceptances are not open. A severity this SDK + * does not know reaches every threshold. Without valid open-finding counts the + * threshold cannot be measured, which is `UNAVAILABLE` and never `NOT_REACHED`. + */ +export function gateReview( + result: ReviewOutcome, + options: GateOptions, +): GateResult { + const failOn = options?.failOn; + if (!SEVERITIES.includes(failOn)) + throw new TypeError("failOn must be WARNING or SERIOUS"); + if (result.status === "TIMED_OUT") return "UNAVAILABLE"; + if (result.status !== "COMPLETED") return "NOT_COMPLETE"; + const review = result.review; + const counts: unknown = review?.open_items; + if ( + !review || + !isObject(counts) || + !isCount(counts.open_findings) || + !isCount(counts.serious_findings) + ) + return "UNAVAILABLE"; + if ( + (failOn === "SERIOUS" ? counts.serious_findings : counts.open_findings) > 0 + ) + return "REACHED"; + // CODE findings have no recorded decisions, so their lifecycle and severity + // decide. A partial CODE assessment's findings are not its whole result. + const code = recordedCode(review); + if ( + code?.status === "COMPLETED" && + !code.incompleteness_kind && + code.findings.some( + (finding) => + finding.lifecycle_state !== "RESOLVED" && + reaches(finding.severity, failOn), + ) + ) + return "REACHED"; + return "NOT_REACHED"; +} + +function isCount(value: unknown): value is number { + return Number.isSafeInteger(value) && Number(value) >= 0; +} + +function reaches(severity: unknown, threshold: SeverityThreshold): boolean { + // An unknown severity, which a newer server can send, is never below one. + const index = SEVERITIES.indexOf(String(severity)); + return index < 0 || index >= SEVERITIES.indexOf(threshold); +} diff --git a/tests/gate.test.mjs b/tests/gate.test.mjs new file mode 100644 index 0000000..5ba5fd1 --- /dev/null +++ b/tests/gate.test.mjs @@ -0,0 +1,254 @@ +import assert from "node:assert/strict"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import test from "node:test"; +import { gateReview, ProoflineClient } from "../dist/index.js"; + +const orgId = "00000000-0000-7000-8000-000000000001"; +const reviewId = "00000000-0000-7000-8000-000000000002"; +const assessmentId = "00000000-0000-7000-8000-000000000003"; +const codeId = "00000000-0000-7000-8000-000000000005"; +const token = "plp_customer_canary_3331"; + +// Serves whatever `body` holds, so each case reads a real `ReviewOutcome`. +async function reader(t) { + const state = { body: null }; + const http = createServer((_, response) => { + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(state.body)); + }); + http.listen(0, "127.0.0.1"); + await once(http, "listening"); + t.after( + () => + new Promise((resolve) => { + http.close(resolve); + http.closeAllConnections(); + }), + ); + const client = new ProoflineClient({ + baseUrl: `http://127.0.0.1:${http.address().port}`, + token, + }); + return async (body, options) => { + state.body = body; + return client.waitForReview(orgId, reviewId, options); + }; +} + +function counts(openFindings = 0, seriousFindings = 0) { + return { + open_findings: openFindings, + serious_findings: seriousFindings, + accepted_risks: 0, + not_assessed: [], + }; +} + +function assessment(status = "COMPLETED") { + return { + id: assessmentId, + environment_display_name: "production", + status, + domains: [], + findings: [], + release_evidence_gap: null, + checks: { receipt_recorded: true, status: "READY", families: [] }, + }; +} + +function review( + status = "COMPLETED", + extra = {}, + assessments = [assessment()], +) { + return { + id: reviewId, + public_key: "review-3331", + kind: "PULL_REQUEST", + status, + assessment_count: assessments.length, + environment_display_names: ["production"], + finding_count: 0, + trigger_origin: "MANUAL", + publication: "WITHHELD", + created_at: "2026-09-30T12:00:00Z", + stoppable: false, + related_reviews: [], + subjects: [], + assessments, + actions: { + promote_findings: false, + rerun: false, + publish: false, + stop: false, + validate_findings: false, + }, + head_commit_sha: "a".repeat(40), + base_commit_sha: "b".repeat(40), + ...extra, + }; +} + +function codeAssessment(findings, status = "COMPLETED", extra = {}) { + return { + id: codeId, + status, + head_commit_sha: "a".repeat(40), + base_commit_sha: "b".repeat(40), + incompleteness_kind: null, + findings, + ...extra, + }; +} + +function codeFinding(severity, lifecycleState = "NEW") { + return { + id: codeId, + claim: "claim", + impact: "impact", + risk_domain: "code", + severity, + lifecycle_state: lifecycleState, + comment_reactions: [], + evidence_refs: [], + rule_refs: [], + symbols: [], + }; +} + +test("completeness decides before the open-finding counts", async (t) => { + const read = await reader(t); + const serious = { open_items: counts(1, 1) }; + assert.equal( + gateReview(await read(review("COMPLETED", serious)), { failOn: "SERIOUS" }), + "REACHED", + ); + const incomplete = await read( + review("COMPLETED", serious, [assessment("INCOMPLETE")]), + ); + assert.equal(incomplete.status, "INCOMPLETE"); + assert.equal(gateReview(incomplete, { failOn: "SERIOUS" }), "NOT_COMPLETE"); + for (const status of ["INCOMPLETE", "FAILED", "STOPPED", "SUPERSEDED"]) { + const result = await read(review(status, serious)); + assert.equal( + gateReview(result, { failOn: "SERIOUS" }), + "NOT_COMPLETE", + status, + ); + } + // A gap decides before the missing counts do. + const gap = await read( + review("COMPLETED", { open_items: null }, [assessment("INCOMPLETE")]), + ); + assert.equal(gateReview(gap, { failOn: "WARNING" }), "NOT_COMPLETE"); + // A wait that ran out has no finished review to measure. + const timedOut = await read(review("RUNNING", serious), { timeoutMs: 50 }); + assert.equal(timedOut.status, "TIMED_OUT"); + assert.equal(gateReview(timedOut, { failOn: "WARNING" }), "UNAVAILABLE"); +}); + +test("a completed review is measured by the server's open-finding counts", async (t) => { + const read = await reader(t); + for (const [openItems, failOn, expected] of [ + [counts(1, 1), "WARNING", "REACHED"], + [counts(1, 1), "SERIOUS", "REACHED"], + [counts(1, 0), "WARNING", "REACHED"], + [counts(1, 0), "SERIOUS", "NOT_REACHED"], + [counts(0, 0), "WARNING", "NOT_REACHED"], + [counts(0, 0), "SERIOUS", "NOT_REACHED"], + // A threshold that cannot be measured is never NOT_REACHED. + [undefined, "WARNING", "UNAVAILABLE"], + [null, "SERIOUS", "UNAVAILABLE"], + [{ open_findings: 0 }, "WARNING", "UNAVAILABLE"], + [counts("0", 0), "WARNING", "UNAVAILABLE"], + [counts(0, -1), "SERIOUS", "UNAVAILABLE"], + [counts(0.5, 0), "WARNING", "UNAVAILABLE"], + [counts(false, 0), "WARNING", "UNAVAILABLE"], + [[], "WARNING", "UNAVAILABLE"], + ]) { + const result = await read(review("COMPLETED", { open_items: openItems })); + assert.equal(result.status, "COMPLETED"); + assert.equal( + gateReview(result, { failOn }), + expected, + `${JSON.stringify(openItems)} ${failOn}`, + ); + } +}); + +test("complete CODE findings of the review's own commits reach by lifecycle and severity", async (t) => { + const read = await reader(t); + for (const [severity, lifecycle, failOn, expected] of [ + ["SERIOUS", "NEW", "SERIOUS", "REACHED"], + ["SERIOUS", "UNCHANGED", "WARNING", "REACHED"], + ["WARNING", "NEW", "WARNING", "REACHED"], + ["WARNING", "NEW", "SERIOUS", "NOT_REACHED"], + ["SERIOUS", "RESOLVED", "SERIOUS", "NOT_REACHED"], + // A severity a newer server sends is never read as below a threshold. + ["CRITICAL", "NEW", "SERIOUS", "REACHED"], + ["NOTE", "NEW", "SERIOUS", "REACHED"], + ]) { + const code = codeAssessment([codeFinding(severity, lifecycle)]); + const case_ = `${severity} ${lifecycle} ${failOn}`; + // Code-only review. + const codeOnly = await read( + review("COMPLETED", { code_assessment: code, open_items: counts() }, []), + ); + assert.equal(codeOnly.status, "COMPLETED", case_); + assert.equal(gateReview(codeOnly, { failOn }), expected, case_); + // Beside complete environments, as the CLI measures it. + const beside = await read( + review("COMPLETED", { code_assessment: code, open_items: counts() }), + ); + assert.equal(gateReview(beside, { failOn }), expected, case_); + } +}); + +test("partial or foreign CODE records supply no findings", async (t) => { + const read = await reader(t); + const serious = [codeFinding("SERIOUS")]; + for (const code of [ + codeAssessment(serious, "INCOMPLETE", { + incompleteness_kind: "MODEL_BUDGET_EXHAUSTED", + }), + codeAssessment(serious, "COMPLETED", { + incompleteness_kind: "EVIDENCE_UNAVAILABLE", + }), + codeAssessment(serious, "RUNNING"), + codeAssessment(serious, "COMPLETED", { head_commit_sha: "c".repeat(40) }), + codeAssessment(serious, "COMPLETED", { base_commit_sha: "c".repeat(40) }), + ]) { + const result = await read( + review("COMPLETED", { code_assessment: code, open_items: counts() }), + ); + assert.equal(result.status, "COMPLETED"); + assert.equal(gateReview(result, { failOn: "WARNING" }), "NOT_REACHED"); + } + // A code-only review with such a record is not complete. + const codeOnly = await read( + review( + "COMPLETED", + { + code_assessment: codeAssessment(serious, "RUNNING"), + open_items: counts(), + }, + [], + ), + ); + assert.equal(gateReview(codeOnly, { failOn: "WARNING" }), "NOT_COMPLETE"); +}); + +test("an invalid threshold throws instead of measuring", async (t) => { + const read = await reader(t); + const result = await read(review("COMPLETED", { open_items: counts() })); + for (const options of [ + { failOn: "warning" }, + { failOn: "CRITICAL" }, + { failOn: undefined }, + {}, + undefined, + ]) { + assert.throws(() => gateReview(result, options), TypeError); + } +});