From 4004ec60b721316031c6a77ff854822d4d1c5049 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Thu, 12 Nov 2026 17:01:15 +0530 Subject: [PATCH] fix(review): verdict trust: only the engine blocks, policy from the base branch, `allowForceTier` - Findings carry `origin: engine | catalog | ai` (from `evidence.tool`; unknown tools count as catalog). Only engine findings count toward REQUEST_CHANGES, matching the documented promise; a pattern rule can be opted in by id via `rubric.blockingRules`. The AI lane never blocks. - `.altimate/review.yml` is read from the merge-base commit, so a PR cannot relax the rules it is judged by. Settings the PR changes are listed in the comment and envelope (`policy.changedInPr`) and take effect after merge. The applied policy's source and hash are signed into the envelope. - `allowForceTier: false` in the base policy rejects `--force-tier` (forced runs were already recorded as `tier_forced` in `review_run`). Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 16 ++ .../opencode/src/altimate/review/config.ts | 100 ++++++++++ .../opencode/src/altimate/review/finding.ts | 27 +++ .../opencode/src/altimate/review/format.ts | 6 + packages/opencode/src/altimate/review/git.ts | 10 + .../src/altimate/review/orchestrate.ts | 3 + .../opencode/src/altimate/review/rubric.ts | 6 + packages/opencode/src/altimate/review/run.ts | 59 ++++-- .../opencode/src/altimate/review/verdict.ts | 39 +++- .../altimate/review-verdict-trust.test.ts | 174 ++++++++++++++++++ .../opencode/test/altimate/review.test.ts | 8 +- .../skill/release-v0.8.1-adversarial.test.ts | 2 + 12 files changed, 420 insertions(+), 30 deletions(-) create mode 100644 packages/opencode/test/altimate/review-verdict-trust.test.ts diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index 7fb829c3b5..75fd31edd8 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -271,9 +271,11 @@ exclude: riskTierPathTokens: # OFF by default — see "Risk-tier path tokens" below finops: [preset:finops] # promote any FinOps-named path to `full` tier pii: [ssn, email, phone] # or supply your own case-insensitive tokens +allowForceTier: true # set false in gate CI: --force-tier is then rejected rubric: blockOn: [lineage_breakage, contract_violation, pii_exposure, semantic_change] warningPatternThreshold: 3 + blockingRules: [] # catalog rule ids allowed to block (default: engine only) thresholds: warehouseCostMinRows: 1000000 exclusions: @@ -281,6 +283,20 @@ rubric: skipNonProdModels: true ``` +### Which policy applies, and what can block + +- **The base branch's `.altimate/review.yml` applies.** The file is read from the + commit the PR branched from, so a PR cannot relax the rules it is reviewed + under (switch to `comment` mode, drop a `blockOn` category, exclude its own + paths). If the PR edits the file, the comment lists the changed settings; they + take effect after merge. The applied policy's hash is signed into the envelope + (`policy`). +- **Only the deterministic engine blocks.** Findings carry an `origin`: + `engine` (AST, lineage, manifest, equivalence, data-diff), `catalog` (pattern + rules) or `ai`. Only `engine` findings count toward REQUEST_CHANGES. A pattern + rule you trust can be allowed to block by naming its id in + `rubric.blockingRules`. The advisory AI lane never blocks. + ### What the PR comment shows - **One summary comment**, updated in place on every push, capped at about diff --git a/packages/opencode/src/altimate/review/config.ts b/packages/opencode/src/altimate/review/config.ts index 9d370d8062..d67cd7462c 100644 --- a/packages/opencode/src/altimate/review/config.ts +++ b/packages/opencode/src/altimate/review/config.ts @@ -2,6 +2,7 @@ import z from "zod" import path from "node:path" import { promises as fs } from "node:fs" import YAML from "yaml" +import { createHash } from "node:crypto" import { Rubric, DEFAULT_RUBRIC } from "./rubric" import { ReviewMode } from "./verdict" import { Severity } from "./finding" @@ -34,6 +35,9 @@ export const ReviewConfig = z.object({ * Functions missing on the project's own engine are always reported. */ portability: z.boolean().optional(), + /** Allow `--force-tier` (a debug bypass of the risk classifier). Set false + * in gate CI so a run cannot skip checks by forcing a lower tier. */ + allowForceTier: z.boolean().default(true), /** * Data-diff: actually run base-vs-head against the warehouse (core DataParity) * and report row/value deltas. OPT-IN — it costs warehouse compute and needs a @@ -114,6 +118,102 @@ export function parseReviewConfig(text: string): ReviewConfig { return ReviewConfig.parse(data) } +/** Where the applied policy came from. */ +export type PolicySource = "base" | "working_tree" | "default" + +export interface ReviewPolicy { + config: ReviewConfig + source: PolicySource + /** Short hash of the applied config (canonical JSON), signed into the envelope. */ + hash: string + /** Settings the PR changes relative to the applied base policy (not applied). */ + changedInPr: string[] +} + +function canonical(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null" + if (Array.isArray(value)) return "[" + value.map(canonical).join(",") + "]" + const obj = value as Record + return ( + "{" + + Object.keys(obj) + .filter((k) => obj[k] !== undefined) + .sort() + .map((k) => JSON.stringify(k) + ":" + canonical(obj[k])) + .join(",") + + "}" + ) +} + +/** Short, stable hash of a resolved config. */ +export function policyHash(config: ReviewConfig): string { + return createHash("sha256").update(canonical(config)).digest("hex").slice(0, 16) +} + +/** Settings that differ between two configs: top-level keys, and `rubric.*` one level down. */ +export function policyChanges(base: ReviewConfig, head: ReviewConfig): string[] { + const out: string[] = [] + const keys = new Set([...Object.keys(base), ...Object.keys(head)]) as Set + for (const k of [...keys].sort()) { + if (canonical(base[k]) === canonical(head[k])) continue + if (k === "rubric") { + const b = (base.rubric ?? {}) as Record + const h = (head.rubric ?? {}) as Record + for (const rk of [...new Set([...Object.keys(b), ...Object.keys(h)])].sort()) + if (canonical(b[rk]) !== canonical(h[rk])) out.push(`rubric.${rk}`) + } else out.push(k) + } + return out +} + +async function readFirst(read: (rel: string) => Promise): Promise { + for (const rel of CANDIDATE_FILES) { + const text = await read(rel) + if (text !== undefined) return text + } + return undefined +} + +/** + * Load the review policy the PR is judged by. + * + * With `readBase` (a reader over the merge-base commit), the policy is the + * BASE branch's `.altimate/review.yml`: a pull request must not be able to + * relax the rules it is reviewed under (switch `mode` to comment, drop a + * `blockOn` category, exclude its own paths, disable lanes). The PR's own + * version is parsed only to list what it would change, which the comment + * shows; it takes effect once merged. Without `readBase` (no git), the working + * tree's file applies. + */ +export async function loadReviewPolicy(opts: { + cwd: string + readBase?: (rel: string) => Promise +}): Promise { + if (!opts.readBase) { + const config = await loadReviewConfig(opts.cwd) + const present = await readFirst((rel) => fs.readFile(path.join(opts.cwd, rel), "utf8").catch(() => undefined)) + return { config, source: present === undefined ? "default" : "working_tree", hash: policyHash(config), changedInPr: [] } + } + const baseText = await readFirst(opts.readBase) + let config: ReviewConfig + try { + config = baseText === undefined ? structuredClone(DEFAULT_REVIEW_CONFIG) : parseReviewConfig(baseText) + } catch (e: any) { + throw new Error(`Failed to load the base branch's .altimate/review config: ${e?.message ?? e}`) + } + const headText = await readFirst((rel) => fs.readFile(path.join(opts.cwd, rel), "utf8").catch(() => undefined)) + let changedInPr: string[] = [] + if (headText !== baseText) { + try { + const head = headText === undefined ? structuredClone(DEFAULT_REVIEW_CONFIG) : parseReviewConfig(headText) + changedInPr = policyChanges(config, head) + } catch { + changedInPr = ["(the PR's .altimate/review file does not parse)"] + } + } + return { config, source: baseText === undefined ? "default" : "base", hash: policyHash(config), changedInPr } +} + /** * Load `.altimate/review.{yml,yaml,json}` from a project directory. Returns the * defaults when no config file is present. Throws on a malformed/invalid file diff --git a/packages/opencode/src/altimate/review/finding.ts b/packages/opencode/src/altimate/review/finding.ts index c9fb1d42e9..41009731a7 100644 --- a/packages/opencode/src/altimate/review/finding.ts +++ b/packages/opencode/src/altimate/review/finding.ts @@ -84,9 +84,36 @@ export const Finding = z.object({ /** True when produced without a manifest/warehouse (lint-only degraded run). */ degraded: z.boolean().default(false), evidence: Evidence.optional(), + /** Which layer produced it: the deterministic engine, the regex/pattern + * catalog, or the advisory AI lane. Only `engine` findings decide the + * verdict unless the rubric opts a named catalog rule in. Stamped by + * buildEnvelope from `evidence.tool`. */ + origin: z.enum(["engine", "catalog", "ai"]).optional(), }) export type Finding = z.infer +export type FindingOrigin = NonNullable + +/** Evidence tools backed by the deterministic engine (AST, manifest, lineage, + * warehouse) rather than text patterns. */ +const ENGINE_TOOLS = new Set(["impact_analysis", "dbt.manifest", "data.diff", "schema.detect_pii"]) + +/** The layer a finding came from. Unknown or missing evidence counts as + * `catalog`, so a new detector can never block until it is classified. */ +export function findingOrigin(f: Pick): FindingOrigin { + if (f.origin) return f.origin + const tool = f.evidence?.tool ?? "" + if (tool === "ai-review") return "ai" + if (tool.startsWith("altimate_core.") || ENGINE_TOOLS.has(tool)) return "engine" + return "catalog" +} + +/** The catalog rule id behind a finding (`evidence.result.rule`), if any. */ +export function catalogRuleId(f: Pick): string | undefined { + const rule = (f.evidence?.result as { rule?: unknown } | undefined)?.rule + return typeof rule === "string" ? rule : undefined +} + /** Fields that define finding identity for fingerprinting / dedup. */ export interface FingerprintInput { category: ReviewCategory | string diff --git a/packages/opencode/src/altimate/review/format.ts b/packages/opencode/src/altimate/review/format.ts index b298b619dc..8f6cf489dd 100644 --- a/packages/opencode/src/altimate/review/format.ts +++ b/packages/opencode/src/altimate/review/format.ts @@ -221,6 +221,12 @@ export function coverageNotes(env: VerdictEnvelope): string[] { `${env.scope.movedUnchangedFiles} file(s) moved without edits; their SQL was not re-reviewed. ` + "Folder-level config in `dbt_project.yml` (`+schema`, `+materialized`, `+tags`) can still change where and how a moved model builds.", ) + if (env.policy?.changedInPr?.length) { + notes.push( + `This PR changes the review policy (\`.altimate/review.yml\`: ${env.policy.changedInPr.join(", ")}). ` + + "The base branch's policy was applied; the change takes effect after merge.", + ) + } if (env.ai && env.ai.state !== "ok") { const what = { skipped: "skipped", timeout: "timed out", error: "failed" }[env.ai.state] notes.push(`Advisory AI review ${what}${env.ai.reason ? `: ${env.ai.reason}` : ""}.`) diff --git a/packages/opencode/src/altimate/review/git.ts b/packages/opencode/src/altimate/review/git.ts index c6087b50bf..2455ea65f9 100644 --- a/packages/opencode/src/altimate/review/git.ts +++ b/packages/opencode/src/altimate/review/git.ts @@ -261,6 +261,16 @@ export async function resolveReviewBase(opts: ResolveBaseOptions): Promise:`), or undefined when the + * path does not exist there. `rel` is repository-relative, POSIX. */ +export async function showAtRef(cwd: string, ref: string, rel: string): Promise { + try { + return await git(["show", `${ref}:${rel}`], cwd) + } catch { + return undefined + } +} + /** Resolve a default base SHA. Kept for callers that only need the commit. */ export async function defaultBaseRef(cwd: string, head?: string): Promise { return (await resolveReviewBase({ cwd, head })).sha diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 515ed2e4d2..643541af87 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -208,6 +208,8 @@ export interface OrchestrateInput { scope?: Omit /** Drop the regex catalog's cross-engine portability findings (single known engine). */ suppressPortability?: boolean + /** Which review policy was applied (see config.ts loadReviewPolicy). */ + policy?: VerdictEnvelope["policy"] } /** A catalog finding tagged as cross-engine portability advice. */ @@ -1536,6 +1538,7 @@ export async function runReview(input: OrchestrateInput): Promise= this many `warning` findings is treated as a risk pattern → block. */ warningPatternThreshold: z.number().int().positive().default(3), + /** + * Catalog (regex/pattern) rule ids allowed to influence the verdict. By + * default only deterministic engine findings can block; a team that trusts a + * specific pattern rule (e.g. `hardcoded-credential`) names it here. + */ + blockingRules: z.array(z.string()).default([]), thresholds: Thresholds.default(Thresholds.parse({})), exclusions: Exclusions.default(Exclusions.parse({})), }) diff --git a/packages/opencode/src/altimate/review/run.ts b/packages/opencode/src/altimate/review/run.ts index 21cdc1ba09..b72c4c3fac 100644 --- a/packages/opencode/src/altimate/review/run.ts +++ b/packages/opencode/src/altimate/review/run.ts @@ -1,7 +1,7 @@ import path from "node:path" import { readFile, realpath, stat, access } from "node:fs/promises" import { Installation } from "../../installation" -import { loadReviewConfig, resolveRubric } from "./config" +import { loadReviewPolicy, resolveRubric } from "./config" import type { Severity } from "./finding" import { collectChangedFiles, @@ -10,6 +10,7 @@ import { type ResolvedBase, gitRepoRoot, manifestHash, + showAtRef, } from "./git" import { makeCompiledResolver, dbtProjectName, dbtProjectLayout } from "./compiled" import { scopeToProject } from "./diff-filter" @@ -217,12 +218,6 @@ async function detectStaleManifest(manifestAbs: string, changed: ChangedFile[], } export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise { - const config = await loadReviewConfig(opts.cwd) - if (opts.manifestPath) config.manifestPath = opts.manifestPath - if (opts.mode) config.mode = opts.mode - if (opts.severityThreshold) config.severityThreshold = opts.severityThreshold - const rubric = resolveRubric(config) - // Resolve the repo top-level once; used to root working-tree FS reads, the // stale-manifest existence check, and the compiled-SQL resolver's path // mapping. Falls back to opts.cwd when we couldn't resolve it (non-git or @@ -230,6 +225,43 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise // in the subdir-invocation case that this addresses. const gitRoot = (await gitRepoRoot(opts.cwd)) ?? opts.cwd + // Only resolve a base ref if we actually need git (to collect changed files + // or to read old/new content). A caller that supplies BOTH `changedFiles` and + // `getContent` (e.g. a non-git CI integration) must not be forced through a + // git lookup that can fail when there's no usable history. + // + // The base is always the MERGE-BASE of the PR's target branch and the head: + // the diff, and every "old" file version, come from the commit the PR + // branched from, so commits the target gained since never leak in as changes. + const needGit = !opts.changedFiles || !opts.getContent + const resolved: ResolvedBase | undefined = needGit + ? await resolveReviewBase({ cwd: opts.cwd, base: opts.base, head: opts.head }) + : undefined + + // The policy (.altimate/review.yml) comes from the BASE commit, so the PR + // under review cannot relax its own rules; what it would change is listed. + const cwdRel = await repoRelative(gitRoot, opts.cwd, true) + const policy = await loadReviewPolicy({ + cwd: opts.cwd, + readBase: + resolved && cwdRel !== undefined + ? (rel) => showAtRef(opts.cwd, resolved.sha, path.posix.join(cwdRel, rel)) + : undefined, + }) + const config = policy.config + if (policy.changedInPr.length) { + process.stderr.write( + `ℹ️ this PR changes .altimate/review (${policy.changedInPr.join(", ")}); the base branch's policy was applied.\n`, + ) + } + if (opts.forceTier && config.allowForceTier === false) { + throw new Error("--force-tier is disabled by `allowForceTier: false` in the base branch's .altimate/review.yml") + } + if (opts.manifestPath) config.manifestPath = opts.manifestPath + if (opts.mode) config.mode = opts.mode + if (opts.severityThreshold) config.severityThreshold = opts.severityThreshold + const rubric = resolveRubric(config) + // Resolve the manifest against the PROJECT being reviewed (cwd), not the // binary's process.cwd() — otherwise a relative path silently misses when the // CLI is invoked from elsewhere, degrading every review to lint-only. @@ -271,18 +303,6 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise const projectRootRel = await repoRelative(gitRoot, dbtRoot, project !== undefined || dbtRoot !== opts.cwd) const layout = projectRootRel !== undefined ? await dbtProjectLayout(dbtRoot, projectRootRel) : undefined - // Only resolve a base ref if we actually need git (to collect changed files - // or to read old/new content). A caller that supplies BOTH `changedFiles` and - // `getContent` (e.g. a non-git CI integration) must not be forced through a - // git lookup that can fail when there's no usable history. - // - // The base is always the MERGE-BASE of the PR's target branch and the head: - // the diff, and every "old" file version, come from the commit the PR - // branched from, so commits the target gained since never leak in as changes. - const needGit = !opts.changedFiles || !opts.getContent - const resolved: ResolvedBase | undefined = needGit - ? await resolveReviewBase({ cwd: opts.cwd, base: opts.base, head: opts.head }) - : undefined if (resolved && resolved.source !== "flag") { process.stderr.write(`ℹ️ review base: ${resolved.ref} (${resolved.source}), merge-base ${resolved.sha.slice(0, 12)}\n`) } @@ -392,6 +412,7 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise staleManifest, layout, suppressPortability: config.portability === undefined ? dialectKnown : !config.portability, + policy: { source: policy.source, hash: policy.hash, changedInPr: policy.changedInPr.length ? policy.changedInPr : undefined }, scope: { baseRef: resolved?.ref ?? opts.base, baseSha: resolved?.sha, diff --git a/packages/opencode/src/altimate/review/verdict.ts b/packages/opencode/src/altimate/review/verdict.ts index 655404de6a..595d8087df 100644 --- a/packages/opencode/src/altimate/review/verdict.ts +++ b/packages/opencode/src/altimate/review/verdict.ts @@ -1,6 +1,6 @@ import z from "zod" import { createHmac, createHash } from "node:crypto" -import { Finding, type Severity } from "./finding" +import { Finding, type Severity, catalogRuleId, findingOrigin } from "./finding" import { Rubric, DEFAULT_RUBRIC, blockingCategories } from "./rubric" /** @@ -41,6 +41,17 @@ export const VCS_EVENT: Record = { REQUEST_CHANGES: "REQUEST_CHANGES", } +/** Whether a finding may influence the verdict: deterministic engine + * findings always; catalog (pattern) findings only when the rubric names the + * rule in `blockingRules`; advisory AI findings never. */ +export function verdictEligible(f: Finding, rubric: Rubric = DEFAULT_RUBRIC): boolean { + const origin = findingOrigin(f) + if (origin === "engine") return true + if (origin === "ai") return false + const rule = catalogRuleId(f) + return !!rule && rubric.blockingRules.includes(rule) +} + /** * Compute the verdict purely from findings + rubric. Faithful to Cloudflare's * bias-toward-approval rubric: @@ -48,21 +59,20 @@ export const VCS_EVENT: Record = { * - >= warningPatternThreshold warnings → REQUEST_CHANGES (risk pattern) * - any finding at all → COMMENT * - nothing → APPROVE + * Only verdict-eligible findings (see verdictEligible) count toward a block: + * the docs promise that only the deterministic engine blocks, so a regex rule + * or the AI lane can comment but never fail the gate on its own. */ export function computeIdealVerdict(findings: Finding[], rubric: Rubric = DEFAULT_RUBRIC): Verdict { if (findings.length === 0) return "APPROVE" const blockers = blockingCategories(rubric) - const hasBlockingCritical = findings.some((f) => f.severity === "critical" && blockers.has(f.category)) + const eligible = findings.filter((f) => verdictEligible(f, rubric)) + const hasBlockingCritical = eligible.some((f) => f.severity === "critical" && blockers.has(f.category)) if (hasBlockingCritical) return "REQUEST_CHANGES" // Count only confidently-warned findings toward the risk pattern. Undecidable // ("unknown") warnings — e.g. equivalence that couldn't be proven — must not // accumulate into a block; that would let unprovable refactors fail the gate. - // Advisory LLM findings (layer 3) are EXCLUDED: the lane is advisory-only and must - // never influence the verdict, so its warnings cannot accumulate into a block - // (otherwise a chatty or prompt-injected AI review could force REQUEST_CHANGES). - const warningCount = findings.filter( - (f) => f.severity === "warning" && f.confidence !== "unknown" && f.evidence?.tool !== "ai-review", - ).length + const warningCount = eligible.filter((f) => f.severity === "warning" && f.confidence !== "unknown").length if (warningCount >= rubric.warningPatternThreshold) return "REQUEST_CHANGES" return "COMMENT" } @@ -144,6 +154,15 @@ export const VerdictEnvelope = z.object({ staleManifest: z.boolean().optional(), /** Base resolution and review coverage (see ReviewScope). */ scope: ReviewScope.optional(), + /** The review policy applied: from the base branch, the working tree, or + * defaults; its hash; and settings the PR changes but that were not applied. */ + policy: z + .object({ + source: z.enum(["base", "working_tree", "default"]), + hash: z.string(), + changedInPr: z.array(z.string()).optional(), + }) + .optional(), /** Advisory AI lane outcome: ran, skipped, timed out or failed, and why. */ ai: z .object({ @@ -213,6 +232,7 @@ export interface BuildEnvelopeInput { staleManifest?: boolean scope?: ReviewScope ai?: VerdictEnvelope["ai"] + policy?: VerdictEnvelope["policy"] } function summarize(findings: Finding[], degraded: boolean): VerdictEnvelope["summary"] { @@ -236,13 +256,14 @@ export function buildEnvelope(input: BuildEnvelopeInput): VerdictEnvelope { tierReasons: input.tierReasons, tierForced: input.tierForced, tierClassified: input.tierClassified, - findings: input.findings, + findings: input.findings.map((f) => (f.origin ? f : { ...f, origin: findingOrigin(f) })), summary: summarize(input.findings, degraded), engine: EngineVersions.parse(input.engine ?? {}), manifestHash: input.manifestHash, staleManifest: input.staleManifest ? true : undefined, scope: input.scope, ai: input.ai, + policy: input.policy, generatedAt: input.generatedAt, }) } diff --git a/packages/opencode/test/altimate/review-verdict-trust.test.ts b/packages/opencode/test/altimate/review-verdict-trust.test.ts new file mode 100644 index 0000000000..2dff815941 --- /dev/null +++ b/packages/opencode/test/altimate/review-verdict-trust.test.ts @@ -0,0 +1,174 @@ +import { describe, test, expect, afterEach } from "bun:test" +import { promises as fs } from "node:fs" +import path from "node:path" +import { tmpdir } from "node:os" +import { execFile } from "node:child_process" +import { promisify } from "node:util" +import { makeFinding, findingOrigin, type Finding } from "../../src/altimate/review/finding" +import { DEFAULT_RUBRIC, Rubric } from "../../src/altimate/review/rubric" +import { buildEnvelope, computeIdealVerdict, signEnvelope, verdictEligible, verifyEnvelope } from "../../src/altimate/review/verdict" +import { DEFAULT_REVIEW_CONFIG, policyChanges, policyHash } from "../../src/altimate/review/config" +import { reviewPullRequest } from "../../src/altimate/review/run" +import { renderSummary } from "../../src/altimate/review/format" + +/** + * A verdict must mean what the docs say: only the deterministic engine blocks + * (pattern rules and the AI lane comment), and a pull request cannot relax the + * policy it is judged by — `.altimate/review.yml` comes from the base branch. + */ + +const exec = promisify(execFile) +const g = (cwd: string, ...args: string[]) => exec("git", args, { cwd }).then((r) => r.stdout.trim()) + +function f(tool: string | undefined, severity: Finding["severity"], category: any, i: number, rule?: string): Finding { + return makeFinding({ + severity, + category, + title: `t${i}`, + body: "b", + file: `models/m${i}.sql`, + confidence: "high", + evidence: tool ? { tool, result: rule ? { rule } : {} } : undefined, + ruleKey: `r${i}`, + }) +} + +describe("only the engine blocks", () => { + test("origin comes from the evidence tool; unknown sources are catalog", () => { + expect(findingOrigin(f("altimate_core.equivalence", "warning", "semantic_change", 1))).toBe("engine") + expect(findingOrigin(f("impact_analysis", "warning", "lineage_breakage", 2))).toBe("engine") + expect(findingOrigin(f("rule-catalog", "warning", "sql_quality", 3))).toBe("catalog") + expect(findingOrigin(f("dbt-patterns", "warning", "sql_quality", 4))).toBe("catalog") + expect(findingOrigin(f("ai-review", "warning", "sql_quality", 5))).toBe("ai") + expect(findingOrigin(f("some.new.detector", "warning", "sql_quality", 6))).toBe("catalog") + expect(findingOrigin(f(undefined, "warning", "sql_quality", 7))).toBe("catalog") + }) + + test("1 catalog critical + 3 catalog warnings → COMMENT", () => { + const findings = [ + f("rule-catalog", "critical", "sql_correctness", 1, "join-no-on"), + f("rule-catalog", "warning", "sql_quality", 2, "a"), + f("dbt-patterns", "warning", "sql_quality", 3, "b"), + f("rule-catalog", "warning", "sql_quality", 4, "c"), + ] + expect(computeIdealVerdict(findings, DEFAULT_RUBRIC)).toBe("COMMENT") + }) + + test("a catalog rule named in blockingRules can block", () => { + const rubric = Rubric.parse({ blockingRules: ["hardcoded-credential"] }) + const cred = f("rule-catalog", "critical", "sql_correctness", 1, "hardcoded-credential") + expect(verdictEligible(cred, rubric)).toBe(true) + expect(computeIdealVerdict([cred], rubric)).toBe("REQUEST_CHANGES") + expect(computeIdealVerdict([cred], DEFAULT_RUBRIC)).toBe("COMMENT") + }) + + test("engine criticals and engine warning patterns still block", () => { + expect(computeIdealVerdict([f("impact_analysis", "critical", "lineage_breakage", 1)])).toBe("REQUEST_CHANGES") + const three = [1, 2, 3].map((i) => f("altimate_core.check", "warning", "sql_quality", i)) + expect(computeIdealVerdict(three)).toBe("REQUEST_CHANGES") + }) + + test("the envelope records each finding's origin and still verifies", () => { + const env = buildEnvelope({ + findings: [f("rule-catalog", "warning", "sql_quality", 1), f("altimate_core.check", "warning", "sql_quality", 2)], + tier: "lite", + mode: "comment", + }) + expect(env.findings.map((x) => x.origin)).toEqual(["catalog", "engine"]) + const signed = signEnvelope(env) + expect(verifyEnvelope(signed)).toBe(true) + // Origin is covered by the signature: flipping it breaks verification. + const tampered = { ...signed, findings: signed.findings.map((x) => ({ ...x, origin: "engine" as const })) } + expect(verifyEnvelope(tampered)).toBe(false) + }) +}) + +describe("policy helpers", () => { + test("hash is stable and changes with the policy", () => { + const a = structuredClone(DEFAULT_REVIEW_CONFIG) + const b = structuredClone(DEFAULT_REVIEW_CONFIG) + expect(policyHash(a)).toBe(policyHash(b)) + b.mode = "gate" + expect(policyHash(a)).not.toBe(policyHash(b)) + }) + + test("changes are listed by setting, rubric one level down", () => { + const base = structuredClone(DEFAULT_REVIEW_CONFIG) + const head = structuredClone(DEFAULT_REVIEW_CONFIG) + head.mode = "gate" + head.exclude = ["models/legacy/**"] + head.rubric = { ...head.rubric, blockOn: ["lineage_breakage"] } + expect(policyChanges(base, head)).toEqual(["exclude", "mode", "rubric.blockOn"]) + }) +}) + +describe("policy comes from the base branch", () => { + const saved = { ...process.env } + afterEach(() => { + for (const k of ["GITHUB_EVENT_PATH", "GITHUB_BASE_REF", "CI_MERGE_REQUEST_TARGET_BRANCH_NAME"]) { + if (saved[k] === undefined) delete process.env[k] + else process.env[k] = saved[k] + } + }) + + async function repo(baseYml: string | undefined, headYml: string | undefined) { + const root = await fs.realpath(await fs.mkdtemp(path.join(tmpdir(), "review-policy-"))) + await g(root, "init", "-q", "-b", "main") + await g(root, "config", "user.email", "t@test") + await g(root, "config", "user.name", "t") + await g(root, "config", "commit.gpgsign", "false") + await fs.mkdir(path.join(root, "models"), { recursive: true }) + await fs.writeFile(path.join(root, "dbt_project.yml"), "name: p\n") + await fs.writeFile(path.join(root, "models", "a.sql"), "select 1 as id\n") + if (baseYml !== undefined) { + await fs.mkdir(path.join(root, ".altimate"), { recursive: true }) + await fs.writeFile(path.join(root, ".altimate", "review.yml"), baseYml) + } + await g(root, "add", "-A") + await g(root, "commit", "-qm", "base") + await g(root, "checkout", "-q", "-b", "feature") + await fs.writeFile(path.join(root, "models", "a.sql"), "select 2 as id\n") + if (headYml !== undefined) { + await fs.mkdir(path.join(root, ".altimate"), { recursive: true }) + await fs.writeFile(path.join(root, ".altimate", "review.yml"), headYml) + } + await g(root, "add", "-A") + await g(root, "commit", "-qm", "pr") + delete process.env.GITHUB_EVENT_PATH + delete process.env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME + process.env.GITHUB_BASE_REF = "main" + return root + } + + test("a PR that relaxes review.yml is still judged by the base policy, and the change is shown", async () => { + const root = await repo("mode: gate\n", "mode: comment\nexclude:\n - models/**\nai: false\n") + const env = await reviewPullRequest({ cwd: root, head: "HEAD", noAi: true }) + expect(env.mode).toBe("gate") + expect(env.policy?.source).toBe("base") + expect(env.policy?.changedInPr).toEqual(["ai", "exclude", "mode"]) + expect(renderSummary(env)).toContain("The base branch's policy was applied") + }, 60_000) + + test("a review.yml the PR adds does not apply until merged", async () => { + const root = await repo(undefined, "mode: gate\n") + const env = await reviewPullRequest({ cwd: root, head: "HEAD", noAi: true }) + expect(env.mode).toBe("comment") + expect(env.policy?.source).toBe("default") + expect(env.policy?.changedInPr).toEqual(["mode"]) + }, 60_000) + + test("an unchanged policy is applied without a note", async () => { + const root = await repo("mode: gate\n", undefined) + const env = await reviewPullRequest({ cwd: root, head: "HEAD", noAi: true }) + expect(env.mode).toBe("gate") + expect(env.policy?.changedInPr).toBeUndefined() + expect(env.policy?.hash).toMatch(/^[0-9a-f]{16}$/) + }, 60_000) + + test("allowForceTier: false in the base policy rejects --force-tier", async () => { + const root = await repo("allowForceTier: false\n", "allowForceTier: true\n") + await expect(reviewPullRequest({ cwd: root, head: "HEAD", noAi: true, forceTier: "trivial" })).rejects.toThrow( + /force-tier is disabled/, + ) + }, 60_000) +}) diff --git a/packages/opencode/test/altimate/review.test.ts b/packages/opencode/test/altimate/review.test.ts index 2ec9445a26..1e7b7833e6 100644 --- a/packages/opencode/test/altimate/review.test.ts +++ b/packages/opencode/test/altimate/review.test.ts @@ -134,6 +134,7 @@ describe("rubric", () => { // verdict.ts // --------------------------------------------------------------------------- describe("verdict", () => { + // Engine findings: only the deterministic engine decides the verdict. const mk = (severity: Finding["severity"], category: any = "sql_quality") => makeFinding({ severity, @@ -141,6 +142,7 @@ describe("verdict", () => { title: "t", body: "b", file: `m_${Math.random()}.sql`, + evidence: { tool: "altimate_core.check", result: {} }, ruleKey: String(Math.random()), }) @@ -2268,7 +2270,8 @@ describe("orchestrate", () => { expect( pii.some((f) => f.evidence?.tool === "dbt-patterns" && (f.evidence?.result as any)?.rule === "pii_into_mart"), ).toBe(true) - expect(env.verdict).toBe("REQUEST_CHANGES") + // A pattern rule is reported but only blocks when the rubric opts it in. + expect(env.verdict).toBe("COMMENT") }) test("zero-target lineage does not mark the file PII-complete, so the regex twin survives", async () => { @@ -2303,7 +2306,8 @@ describe("orchestrate", () => { expect( pii.some((f) => f.evidence?.tool === "dbt-patterns" && (f.evidence?.result as any)?.rule === "pii_into_mart"), ).toBe(true) - expect(env.verdict).toBe("REQUEST_CHANGES") + // A pattern rule is reported but only blocks when the rubric opts it in. + expect(env.verdict).toBe("COMMENT") }) test("core considered the matched column (muted as low-confidence) → the regex twin is suppressed", async () => { diff --git a/packages/opencode/test/skill/release-v0.8.1-adversarial.test.ts b/packages/opencode/test/skill/release-v0.8.1-adversarial.test.ts index e35f3886e3..bf958b3302 100644 --- a/packages/opencode/test/skill/release-v0.8.1-adversarial.test.ts +++ b/packages/opencode/test/skill/release-v0.8.1-adversarial.test.ts @@ -76,6 +76,7 @@ describe("v0.8.1 — no formal GitHub APPROVE (review bot)", () => { title: "drops rows", body: "the refactor is not equivalent", file: "models/m.sql", + evidence: { tool: "altimate_core.equivalence" }, }) const env = buildEnvelope({ findings: [finding], tier: "full", mode: "comment" }) expect(env.idealVerdict).toBe("REQUEST_CHANGES") // would have blocked @@ -91,6 +92,7 @@ describe("v0.8.1 — no formal GitHub APPROVE (review bot)", () => { title: "new PII column", body: "exposes email", file: "models/users.sql", + evidence: { tool: "altimate_core.classify_pii" }, }) const env = buildEnvelope({ findings: [finding], tier: "full", mode: "gate" }) expect(env.verdict).toBe("REQUEST_CHANGES")