diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index 75fd31edd8..78e5df7115 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -108,7 +108,7 @@ Options: | Flag | Description | |------|-------------| -| `--base ` | Base git ref. Defaults to the pull request's target branch in CI (GitHub `pull_request.base.ref` or `GITHUB_BASE_REF`, GitLab `CI_MERGE_REQUEST_TARGET_BRANCH_NAME`), else the remote default branch. Always reduced to the merge-base with the head, so commits the base gained after the PR branched are never reviewed. In CI, a target branch that is not fetched is an error (use `fetch-depth: 0`). | +| `--base ` | Base git ref. Defaults to the pull request's target branch in CI (GitHub `pull_request.base.ref` or `GITHUB_BASE_REF`, GitLab `CI_MERGE_REQUEST_TARGET_BRANCH_NAME`), else the remote default branch. Always reduced to the merge-base with the head, so commits the base gained after the PR branched are never reviewed. In CI, a shallow checkout is handled: the target branch is fetched and the history deepened until the merge-base is found. If that fails (no `origin`, no access), the review stops with an error rather than diffing against another branch. | | `--head ` | Head git ref. Omit to review the working tree. | | `--manifest ` | Path to the compiled `manifest.json`. When omitted, the reviewer walks up from the current directory to find the nearest `dbt_project.yml` and uses its adjacent `target/manifest.json`; the discovered path is logged to stderr. | | `--mode comment\|gate` | `comment` never blocks; `gate` exits non-zero on `REQUEST_CHANGES`. | @@ -203,7 +203,7 @@ jobs: | Input | How to produce it | Unlocks | |---|---|---| -| Full git history of the PR's target branch | `actions/checkout` with `fetch-depth: 0` | Correct base: the diff starts at the merge-base with the PR's target branch. A target branch that is not fetched is an error, never a silent fallback to `main`. | +| The PR's target branch and the merge-base | Fetched automatically from `origin` when the checkout is shallow; `fetch-depth: 0` avoids the extra fetches | Correct base: the diff starts at the merge-base with the PR's target branch. When it can't be fetched, the review stops with an error, never a silent fallback to `main`. | | `target/manifest.json` | `dbt compile` (or `dbt parse` for lint + lineage only) | Blast radius, lineage breakage, tiering. Without it the run is lint-only. | | `target/compiled/…` | `dbt compile` | Engine checks on the head (lint, grade, PII, grain). | | `target-base/compiled/…` | Compile the base ref into `target-base/` (below) | Equivalence proofs and column-breakage for modified models. | @@ -296,6 +296,12 @@ rubric: rules) or `ai`. Only `engine` findings count toward REQUEST_CHANGES. A pattern rule you trust can be allowed to block by naming its id in `rubric.blockingRules`. The advisory AI lane never blocks. +- **Upgrading a `gate` setup.** Before this policy, any critical finding in a + `blockOn` category could block, including pattern rules such as the regex + PII check. Now pattern-rule findings comment but don't block. To keep a + pattern rule blocking, add its id to `rubric.blockingRules`. A blocking + review is dismissed only by a later run that completed: when an engine call + or a file failed, the earlier block stays. ### What the PR comment shows @@ -311,7 +317,14 @@ rubric: fingerprint, so a push does not repeat findings already on the PR, and a non-blocking rerun with nothing new only updates the summary. - **PII:** flagged only for PII columns the change newly exposes, not for - columns the base already surfaced. + columns the base already surfaced. A model the PII check could not analyse is + listed as not fully verified. +- **Failures are never a clean pass:** if an engine check throws, the comment + names it and the verdict is at most COMMENT, never APPROVE (envelope + `engineErrors`). A failure on one file marks that file unverified and the rest + of the PR is still reviewed. +- **The gate clears itself:** when a rerun no longer blocks, the earlier blocking + altimate review is dismissed (needs `pull-requests: write`). ### Risk-tier path tokens diff --git a/packages/opencode/src/altimate/review/format.ts b/packages/opencode/src/altimate/review/format.ts index 8f6cf489dd..0ae0c40acb 100644 --- a/packages/opencode/src/altimate/review/format.ts +++ b/packages/opencode/src/altimate/review/format.ts @@ -206,13 +206,19 @@ export function coverageNotes(env: VerdictEnvelope): string[] { const undecided = env.findings.filter((f) => f.degraded).length if (undecided) notes.push(`${undecided} finding(s) could not be decided by the engine and are marked _unverified_.`) const unverified = env.scope?.unverified ?? [] - if (unverified.length) { - const head = unverified.slice(0, 5).map((u) => `\`${u.path}\``).join(", ") - const more = unverified.length > 5 ? ` and ${unverified.length - 5} more` : "" - notes.push( - `${unverified.length} model(s) not verified by the engine — ${unverified[0].reason}: ${head}${more}. ` + - "Run `dbt compile` (and compile the base into `target-base/`) for equivalence and lineage checks.", - ) + const byReason = new Map() + for (const u of unverified) byReason.set(u.reason, [...(byReason.get(u.reason) ?? []), u.path]) + for (const [reason, paths] of byReason) { + const head = paths.slice(0, 5).map((p) => `\`${p}\``).join(", ") + const more = paths.length > 5 ? ` and ${paths.length - 5} more` : "" + const hint = /compiled SQL/.test(reason) + ? " Run `dbt compile` (and compile the base into `target-base/`) for equivalence and lineage checks." + : "" + notes.push(`${paths.length} model(s) not fully verified — ${reason}: ${head}${more}.${hint}`) + } + if (env.engineErrors?.length) { + const list = env.engineErrors.map((e) => `${e.method} ×${e.count} (${e.reason})`).join("; ") + notes.push(`Some engine checks failed and did not run: ${list}. This verdict is not a clean approval.`) } if (env.scope?.outOfScopeFiles) notes.push(`${env.scope.outOfScopeFiles} changed file(s) outside the dbt project were not reviewed.`) diff --git a/packages/opencode/src/altimate/review/git.ts b/packages/opencode/src/altimate/review/git.ts index 2455ea65f9..5dd6ee7f65 100644 --- a/packages/opencode/src/altimate/review/git.ts +++ b/packages/opencode/src/altimate/review/git.ts @@ -191,6 +191,33 @@ async function mergeBase(a: string, b: string, cwd: string): Promise Promise): Promise { + if (branch.startsWith("-")) return false + try { + await git(["fetch", "--no-tags", "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`], cwd) + } catch { + return false + } + if (await found()) return true + for (const depth of [50, 200, 1000]) { + try { + if ((await git(["rev-parse", "--is-shallow-repository"], cwd)).trim() !== "true") return false + const headSha = (await git(["rev-parse", "HEAD"], cwd)).trim() + await git(["fetch", "--no-tags", `--deepen=${depth}`, "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`, headSha], cwd) + } catch { + return false + } + if (await found()) return true + } + return false +} + /** The PR's target branch from the CI provider, when the run is for a PR/MR. */ async function ciTargetBranch(env: NodeJS.ProcessEnv): Promise<{ branch: string; source: BaseSource } | undefined> { if (env.GITHUB_EVENT_PATH) { @@ -230,10 +257,15 @@ export async function resolveReviewBase(opts: ResolveBaseOptions): Promise { + for (const ref of [`origin/${target.branch}`, target.branch]) { + const sha = await mergeBase(ref, head, opts.cwd) + if (sha) return { ref, sha, source: target.source } + } + return undefined } + const found = (await find()) ?? ((await fetchTargetHistory(target.branch, opts.cwd, find)) ? await find() : undefined) + if (found) return found throw new Error( `PR target branch '${target.branch}' (${target.source}) is not available locally. ` + `Fetch it (actions/checkout with fetch-depth: 0, or git fetch origin ${target.branch}) or pass --base.`, diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 643541af87..05b7f5f306 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -101,6 +101,8 @@ export interface GradeResult { export interface CheckResult { issues?: Array<{ rule: string; message: string; line?: number; severity?: string; category?: string }> piiColumns?: string[] + /** Core's PII pass could not parse the SQL: `piiColumns` is unknown, not empty. */ + piiAbstained?: boolean /** * True when the native core lint actually ran (parsed the SQL via the AST), * even if it found zero issues. Distinguishes "clean SQL" from "core @@ -118,7 +120,10 @@ export interface ReviewRunner { grade(sql: string, dialect: string): Promise check(sql: string, dialect: string, baseSql?: string): Promise equivalence(oldSql: string, newSql: string, dialect?: string): Promise - detectPii(sql: string, dialect: string): Promise<{ columns: string[] }> + /** `abstained`: the PII pass could not analyse the SQL, so `columns` is unknown. */ + detectPii(sql: string, dialect: string): Promise<{ columns: string[]; abstained?: boolean }> + /** Engine calls that threw during this review (method, first reason, count). */ + failures?(): Array<{ method: string; reason: string; count: number }> /** * Lexical scan (reserved-word aliases + dialect operators) — the curated lists * and detection live in the compiled core; this passes raw added diff lines. @@ -1147,35 +1152,55 @@ export async function runReview(input: OrchestrateInput): Promise false) } + // PR-level lanes (across files) that threw: reported with the engine errors. + const laneErrors: Array<{ method: string; reason: string; count: number }> = [] + const prLevel = (method: string, lane: Promise) => + lane.catch((err) => { + laneErrors.push({ method, reason: (err instanceof Error ? err.message : String(err)).slice(0, 200), count: 1 }) + return [] as Finding[] + }) + const failedFile = (file: { path: string }, err: unknown) => + unverified.push({ + path: file.path, + reason: `review failed for this file: ${(err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 160)}`, + }) await Promise.all( modelFiles.map(async (file) => { - const model = modelNameFromPath(file.path) - // For renames the previous content lives at oldPath, not the new path. - const oldRef = file.oldPath ?? file.path - const withOld = hasOldSide(file) - const [newSql, oldSql, compiledNew, compiledOld] = await Promise.all([ - file.status !== "deleted" ? getContent?.(file.path, "new") : Promise.resolve(undefined), - withOld ? getContent?.(oldRef, "old") : Promise.resolve(undefined), - file.status !== "deleted" ? getCompiled?.(file.path, "new") : Promise.resolve(undefined), - withOld ? getCompiled?.(oldRef, "old") : Promise.resolve(undefined), - ]) - // Engine lanes need dbt-compiled SQL. Raw SQL is used only when it holds - // no Jinja: the AST engine reads `{{ config() }}` as an injection attempt - // and a mixed raw-vs-compiled pair can never be proven equivalent, so a - // Jinja model without compiled output is marked unverified instead of - // producing warnings about its template syntax. The dbt-patterns lane - // always uses raw (it needs the Jinja). - const engineNewSql = compiledNew ?? (file.kind === "model_sql" && hasJinja(newSql) ? undefined : newSql) - const engineOldSql = compiledOld ?? (file.kind === "model_sql" && hasJinja(oldSql) ? undefined : oldSql) - if (file.status !== "deleted" && newSql && !engineNewSql) - unverified.push({ path: file.path, reason: "no compiled SQL for the head (run dbt compile)" }) - else if (withOld && oldSql && !engineOldSql) - unverified.push({ path: file.path, reason: "no compiled SQL for the base (compile the base into target-base)" }) - const impact = await input.runner.impact(model) - if (impact.hasManifest) anyManifest = true - const pii = engineNewSql ? (await input.runner.detectPii(engineNewSql, dialect)).columns : [] - const complex = engineNewSql && input.runner.isComplex ? await input.runner.isComplex(engineNewSql) : false - ctxByPath.set(file.path, { file, impact, pii, newSql, oldSql, engineNewSql, engineOldSql, complex }) + try { + const model = modelNameFromPath(file.path) + // For renames the previous content lives at oldPath, not the new path. + const oldRef = file.oldPath ?? file.path + const withOld = hasOldSide(file) + const [newSql, oldSql, compiledNew, compiledOld] = await Promise.all([ + file.status !== "deleted" ? getContent?.(file.path, "new") : Promise.resolve(undefined), + withOld ? getContent?.(oldRef, "old") : Promise.resolve(undefined), + file.status !== "deleted" ? getCompiled?.(file.path, "new") : Promise.resolve(undefined), + withOld ? getCompiled?.(oldRef, "old") : Promise.resolve(undefined), + ]) + // Engine lanes need dbt-compiled SQL. Raw SQL is used only when it holds + // no Jinja: the AST engine reads `{{ config() }}` as an injection attempt + // and a mixed raw-vs-compiled pair can never be proven equivalent, so a + // Jinja model without compiled output is marked unverified instead of + // producing warnings about its template syntax. The dbt-patterns lane + // always uses raw (it needs the Jinja). + const engineNewSql = compiledNew ?? (file.kind === "model_sql" && hasJinja(newSql) ? undefined : newSql) + const engineOldSql = compiledOld ?? (file.kind === "model_sql" && hasJinja(oldSql) ? undefined : oldSql) + if (file.status !== "deleted" && newSql && !engineNewSql) + unverified.push({ path: file.path, reason: "no compiled SQL for the head (run dbt compile)" }) + else if (withOld && oldSql && !engineOldSql) + unverified.push({ path: file.path, reason: "no compiled SQL for the base (compile the base into target-base)" }) + const impact = await input.runner.impact(model) + if (impact.hasManifest) anyManifest = true + const piiRes = engineNewSql ? await input.runner.detectPii(engineNewSql, dialect) : { columns: [] } + if (piiRes.abstained) unverified.push({ path: file.path, reason: "the PII check could not analyse this model" }) + const pii = piiRes.columns + const complex = engineNewSql && input.runner.isComplex ? await input.runner.isComplex(engineNewSql) : false + ctxByPath.set(file.path, { file, impact, pii, newSql, oldSql, engineNewSql, engineOldSql, complex }) + } catch (err) { + // One file's failure (unreadable content, an engine crash) degrades that + // file only; it must not abort the whole review. + failedFile(file, err) + } }), ) @@ -1327,20 +1352,32 @@ export async function runReview(input: OrchestrateInput): Promise + t.catch((err) => { + if (!laneFailed) failedFile(ctx.file, err) + laneFailed = true + return [] as Finding[] + }), + ), + ) + all.push(...settled) } // PR-level: cross-model sibling filter-consistency (divergent filters on a // shared upstream across the changed models). if (lanes.has("semantic_change")) { - all.push(await siblingConsistencyLane([...ctxByPath.values()], input.runner)) + all.push(await prLevel("sibling_consistency", siblingConsistencyLane([...ctxByPath.values()], input.runner))) } // PR-level: topology-aware equivalence for merge refactors (N models deleted + // 1 added that supersedes them). Per-model equivalence can't see these because // the new model has no 1:1 base. Advisory only (capped at warning). if (lanes.has("semantic_change")) { - all.push(await topologyEquivalenceLane(modelFiles, input.runner, getCompiled, dialect)) + all.push(await prLevel("topology_equivalence", topologyEquivalenceLane(modelFiles, input.runner, getCompiled, dialect))) } // schema.yml-level detectors (test removal) — run on changed YAML files @@ -1393,7 +1430,9 @@ export async function runReview(input: OrchestrateInput): Promise { if (!ctx.engineNewSql || ctx.file.status === "deleted") return // Cache hit: qualityLane already ran this exact check. - const cr = await input.runner.check(ctx.engineNewSql, dialect, ctx.engineOldSql) + const cr = await input.runner + .check(ctx.engineNewSql, dialect, ctx.engineOldSql) + .catch(() => ({ ran: false }) as CheckResult) if (cr.ran) coreRanFiles.add(ctx.file.path) }), ) @@ -1536,7 +1575,9 @@ export async function runReview(input: OrchestrateInput): Promise u.reason.startsWith("review failed for this file")) +} + const BOT_REVIEW_RE = /(Approved|Reviewed with comments|Changes requested)\b.*\((trivial|lite|full) tier/ -export async function postGitHubReview(env: VerdictEnvelope, target: GitHubTarget): Promise { - const octo = new Octokit({ auth: target.token }) +/** The Octokit surface the poster uses; injectable so the posting rules + * (dedup, dismissal, skip) are testable without the network. */ +export type GitHubClient = Pick & { + rest: Pick +} + +export async function postGitHubReview( + env: VerdictEnvelope, + target: GitHubTarget, + client?: GitHubClient, +): Promise { + const octo = client ?? new Octokit({ auth: target.token }) const { owner, repo, prNumber } = target const result: PostResult = { inlineFellBack: false } @@ -99,13 +117,16 @@ export async function postGitHubReview(env: VerdictEnvelope, target: GitHubTarge // ignores line numbers) is not posted again on the next push. let posted = new Set() let priorBotReview = false + let staleBlocking: number[] = [] try { const [reviewComments, reviews] = await Promise.all([ octo.paginate(octo.rest.pulls.listReviewComments, { owner, repo, pull_number: prNumber, per_page: 100 }), octo.paginate(octo.rest.pulls.listReviews, { owner, repo, pull_number: prNumber, per_page: 100 }), ]) posted = postedFindingIds(reviewComments.map((c) => c.body)) - priorBotReview = reviews.some((r) => BOT_REVIEW_RE.test(r.body ?? "")) + const botReviews = reviews.filter((r) => BOT_REVIEW_RE.test(r.body ?? "")) + priorBotReview = botReviews.length > 0 + staleBlocking = botReviews.filter((r) => r.state === "CHANGES_REQUESTED").map((r) => r.id) } catch { /* listing failed: post everything rather than nothing */ } @@ -113,6 +134,30 @@ export async function postGitHubReview(env: VerdictEnvelope, target: GitHubTarge const withLine = env.findings.filter((f) => typeof f.startLine === "number").length result.inlineSkippedAsPosted = withLine - comments.length + // Gate lifecycle: when this run no longer blocks, an earlier blocking + // altimate review would keep the PR blocked forever. Dismiss it, pointing at + // the new verdict. (Requires `pull-requests: write`; a failure is non-fatal.) + // Only a run that actually completed may clear a block: when an engine call + // or a file failed, the blocking finding may simply not have been produced. + if (event === "COMMENT" && staleBlocking.length && runCompleted(env)) { + let dismissed = 0 + for (const id of staleBlocking) { + try { + await octo.rest.pulls.dismissReview({ + owner, + repo, + pull_number: prNumber, + review_id: id, + message: `Superseded by a newer altimate review: ${reviewBody}`, + }) + dismissed++ + } catch { + /* keep going; the summary still shows the current verdict */ + } + } + result.dismissedReviews = dismissed + } + // Nothing new inline, not blocking, and the PR already has an altimate // review: the updated summary comment says it all; another "Reviewed with // comments" entry per push is noise. A blocking verdict always posts. diff --git a/packages/opencode/src/altimate/review/runner.ts b/packages/opencode/src/altimate/review/runner.ts index eec3732d81..aa59b398ca 100644 --- a/packages/opencode/src/altimate/review/runner.ts +++ b/packages/opencode/src/altimate/review/runner.ts @@ -102,6 +102,16 @@ export interface DispatcherRunnerOptions { export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRunner { const checkCache = new Map() + // Engine calls that threw, by method → first reason. Each method still + // returns its empty result so one failure never aborts the review, but the + // failures are reported (envelope `engineErrors`, the comment, telemetry) + // instead of reading as "no findings". + const failures = new Map() + function fail(method: string, err: unknown) { + const reason = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 200) + const prev = failures.get(method) + failures.set(method, { reason: prev?.reason ?? reason, count: (prev?.count ?? 0) + 1 }) + } let manifestPromise: Promise | undefined async function loadManifest(): Promise { @@ -252,8 +262,14 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun ].filter((c): c is string => typeof c === "string" && !!c) // ran=true: the core parsed and analyzed the SQL (even if zero issues). // This lets the orchestrator defer structural checks to the AST lint. - out = { issues, piiColumns: [...new Set(piiColumns)], ran: true } - } catch { + out = { + issues, + piiColumns: [...new Set(piiColumns)], + ran: true, + piiAbstained: !!(data.pii as any)?.parse_error, + } + } catch (err) { + fail("check", err) out = { issues: [], piiColumns: [], ran: false } } checkCache.set(cacheKey, out) @@ -310,7 +326,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.grade", { sql, schema_context: await resolveSchema() }) const data = (res.data ?? {}) as Record return { grade: data.overall_grade ?? data.grade } - } catch { + } catch (err) { + fail("grade", err) return {} } }, @@ -324,7 +341,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.grain", { sql }) const d = (res.data ?? {}) as Record return { group_by: asArray(d.group_by), dedup_partition: asArray(d.dedup_partition) } - } catch { + } catch (err) { + fail("grain", err) return { group_by: [], dedup_partition: [] } } }, @@ -337,14 +355,16 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun try { const res = await Dispatcher.call("altimate_core.dbt_config_lint", { sql: rawSql }) out.push(...asArray((res.data as any)?.findings)) - } catch { + } catch (err) { + fail("dbtConfigLint", err) /* skip */ } if (oldRawSql) { try { const res = await Dispatcher.call("altimate_core.dbt_config_diff", { base_sql: oldRawSql, head_sql: rawSql }) out.push(...asArray((res.data as any)?.findings)) - } catch { + } catch (err) { + fail("dbtConfigLint", err) /* skip */ } } @@ -358,7 +378,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun try { const res = await Dispatcher.call("altimate_core.structural_diff", { base_sql: baseSql, head_sql: headSql }) return asArray((res.data as any)?.findings) - } catch { + } catch (err) { + fail("structuralDiff", err) return [] } }, @@ -368,7 +389,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.source_filters", { sql }) const d = (res.data ?? {}) as Record return (d.filters ?? {}) as Record - } catch { + } catch (err) { + fail("sourceFilters", err) return {} } }, @@ -397,7 +419,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.metadata", { sql }) const d = (res.data ?? {}) as Record return !!(d.has_window_functions || d.has_subqueries || (typeof d.node_count === "number" && d.node_count >= 12)) - } catch { + } catch (err) { + fail("isComplex", err) return false } }, @@ -423,7 +446,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun confidence: typeof c.confidence === "number" ? c.confidence : 0, masking: c.suggested_masking ?? undefined, })) - } catch { + } catch (err) { + fail("classifyPii", err) return [] } }, @@ -450,7 +474,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun rowsChanged: o.rows_changed ?? o.different ?? o.changed, summary: o.summary ?? r.summary, } - } catch { + } catch (err) { + fail("dataDiff", err) return null } }, @@ -468,7 +493,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun source: String(e.source ?? ""), target: String(e.target ?? ""), })) - } catch { + } catch (err) { + fail("columnLineage", err) return [] } }, @@ -484,7 +510,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun message: String(f.message ?? ""), line: f.line, })) - } catch { + } catch (err) { + fail("lexicalScan", err) return [] } }, @@ -529,16 +556,24 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun differences: diffs.map((d) => d?.description ?? String(d)), confidence: EngineCoerce.bandConfidence(data.confidence), } - } catch { + } catch (err) { + fail("equivalence", err) return { decided: false } } }, - async detectPii(sql: string): Promise<{ columns: string[] }> { + async detectPii(sql: string): Promise<{ columns: string[]; abstained?: boolean }> { // Text-based PII comes from altimate_core.check (schema.detect_pii is // warehouse-based and unavailable in CI). Reuses the memoized check. + // When core could not analyse the SQL (check did not run, or its PII + // parser reported a parse error) the empty list means "unknown", not + // "no PII": say so. const result = await runCheck(sql) - return { columns: result.piiColumns ?? [] } + return { columns: result.piiColumns ?? [], abstained: !result.ran || result.piiAbstained === true } + }, + + failures() { + return [...failures].map(([method, f]) => ({ method, reason: f.reason, count: f.count })) }, } } diff --git a/packages/opencode/src/altimate/review/telemetry.ts b/packages/opencode/src/altimate/review/telemetry.ts index 3fbe4af1bb..0e6f4c0ad5 100644 --- a/packages/opencode/src/altimate/review/telemetry.ts +++ b/packages/opencode/src/altimate/review/telemetry.ts @@ -117,6 +117,7 @@ export function emitReviewRun(input: { ai_duration_ms: env.ai?.durationMs, base_source: env.scope?.baseSource, out_of_scope_files: env.scope?.outOfScopeFiles ?? 0, + engine_errors: env.engineErrors?.reduce((n, e) => n + e.count, 0) ?? 0, stale_manifest: env.staleManifest === true, critical: env.summary.critical, warning: env.summary.warning, diff --git a/packages/opencode/src/altimate/review/verdict.ts b/packages/opencode/src/altimate/review/verdict.ts index 595d8087df..c8f510dbb9 100644 --- a/packages/opencode/src/altimate/review/verdict.ts +++ b/packages/opencode/src/altimate/review/verdict.ts @@ -154,6 +154,11 @@ export const VerdictEnvelope = z.object({ staleManifest: z.boolean().optional(), /** Base resolution and review coverage (see ReviewScope). */ scope: ReviewScope.optional(), + /** Engine calls that failed during the review. Their checks did not run, so + * the verdict is never a clean APPROVE when this is present. */ + engineErrors: z + .array(z.object({ method: z.string(), reason: z.string(), count: z.number().int().positive() })) + .optional(), /** The review policy applied: from the base branch, the working tree, or * defaults; its hash; and settings the PR changes but that were not applied. */ policy: z @@ -233,6 +238,7 @@ export interface BuildEnvelopeInput { scope?: ReviewScope ai?: VerdictEnvelope["ai"] policy?: VerdictEnvelope["policy"] + engineErrors?: VerdictEnvelope["engineErrors"] } function summarize(findings: Finding[], degraded: boolean): VerdictEnvelope["summary"] { @@ -244,7 +250,10 @@ function summarize(findings: Finding[], degraded: boolean): VerdictEnvelope["sum /** Assemble the verdict envelope (unsigned). Call signEnvelope to sign it. */ export function buildEnvelope(input: BuildEnvelopeInput): VerdictEnvelope { const rubric = input.rubric ?? DEFAULT_RUBRIC - const ideal = computeIdealVerdict(input.findings, rubric) + // An engine failure means some checks did not run: "no findings" is then + // unknown, not clean, so the verdict cannot be APPROVE. + const computed = computeIdealVerdict(input.findings, rubric) + const ideal = computed === "APPROVE" && input.engineErrors?.length ? "COMMENT" : computed const verdict = applyMode(ideal, input.mode) const degraded = input.degraded ?? input.findings.some((f) => f.degraded) return VerdictEnvelope.parse({ @@ -264,6 +273,7 @@ export function buildEnvelope(input: BuildEnvelopeInput): VerdictEnvelope { scope: input.scope, ai: input.ai, policy: input.policy, + engineErrors: input.engineErrors, generatedAt: input.generatedAt, }) } diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 8566507850..288b69319a 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -1608,6 +1608,8 @@ export namespace Telemetry { /** Where the review base came from (flag, github_event, default_branch, …). */ base_source?: string out_of_scope_files?: number + /** Engine calls that threw during the review (their checks did not run). */ + engine_errors?: number stale_manifest?: boolean critical?: number warning?: number diff --git a/packages/opencode/src/cli/cmd/review.ts b/packages/opencode/src/cli/cmd/review.ts index cef6942c20..b1677b7fe1 100644 --- a/packages/opencode/src/cli/cmd/review.ts +++ b/packages/opencode/src/cli/cmd/review.ts @@ -194,7 +194,8 @@ export const ReviewCommand = cmd({ `Posted review to ${where}` + (r.inlineFellBack ? " (inline comments fell back to summary-only)" : "") + (r.inlineSkippedAsPosted ? ` (${r.inlineSkippedAsPosted} inline already on the PR)` : "") + - (r.reviewSkipped ? " (summary updated; no new review needed)" : ""), + (r.reviewSkipped ? " (summary updated; no new review needed)" : "") + + (r.dismissedReviews ? ` (dismissed ${r.dismissedReviews} earlier blocking review(s))` : ""), ) } } diff --git a/packages/opencode/test/altimate/review-failures-gate.test.ts b/packages/opencode/test/altimate/review-failures-gate.test.ts new file mode 100644 index 0000000000..3fcc45fc79 --- /dev/null +++ b/packages/opencode/test/altimate/review-failures-gate.test.ts @@ -0,0 +1,260 @@ +import { describe, test, expect } from "bun:test" +import { runReview, type ReviewRunner } from "../../src/altimate/review/orchestrate" +import { DEFAULT_REVIEW_CONFIG } from "../../src/altimate/review/config" +import { DEFAULT_RUBRIC } from "../../src/altimate/review/rubric" +import { makeFinding } from "../../src/altimate/review/finding" +import { buildEnvelope, signEnvelope, verifyEnvelope } from "../../src/altimate/review/verdict" +import { renderSummary, findingMarker } from "../../src/altimate/review/format" +import { postGitHubReview, type GitHubClient } from "../../src/altimate/review/post-github" + +/** + * A review must never look clean because something broke: engine failures and + * PII abstentions are reported, one file's crash degrades only that file, and + * a blocking review is cleared when a rerun no longer blocks. + */ + +function runner(over: Partial = {}): ReviewRunner { + return { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return { grade: "B" } + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: true, equivalent: true } + }, + async detectPii() { + return { columns: [] } + }, + ...over, + } as ReviewRunner +} + +const two = [ + { path: "models/a.sql", status: "added" as const, diff: "+select 1 as id\n" }, + { path: "models/b.sql", status: "added" as const, diff: "+select 2 as id\n" }, +] + +describe("engine failures are never clean", () => { + test("no findings plus an engine failure is COMMENT, not APPROVE, and the failure is shown", async () => { + const env = await runReview({ + changedFiles: two, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ failures: () => [{ method: "check", reason: "native core not loaded", count: 2 }] }), + getContent: async (f) => (f.endsWith("a.sql") ? "select 1 as id" : "select 2 as id"), + }) + expect(env.findings).toHaveLength(0) + expect(env.idealVerdict).toBe("COMMENT") + expect(env.engineErrors).toEqual([{ method: "check", reason: "native core not loaded", count: 2 }]) + expect(renderSummary(env)).toContain("Some engine checks failed and did not run: check ×2 (native core not loaded)") + }) + + test("engine errors are signed", () => { + const env = signEnvelope( + buildEnvelope({ findings: [], tier: "lite", mode: "comment", engineErrors: [{ method: "equivalence", reason: "x", count: 1 }] }), + ) + expect(env.verdict).toBe("COMMENT") + expect(verifyEnvelope(env)).toBe(true) + expect(verifyEnvelope({ ...env, engineErrors: undefined })).toBe(false) + }) + + test("without failures an empty review still approves", () => { + expect(buildEnvelope({ findings: [], tier: "lite", mode: "comment" }).verdict).toBe("APPROVE") + }) +}) + +describe("one file's failure degrades only that file", () => { + test("unreadable content for one model: the other is still reviewed", async () => { + let checked: string[] = [] + const env = await runReview({ + changedFiles: two, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ + async check(sql: string) { + checked.push(sql) + return { issues: [], ran: true } + }, + }), + getContent: async (f) => { + if (f.endsWith("a.sql")) throw new Error("EACCES: permission denied") + return "select 2 as id" + }, + }) + expect(checked).toContain("select 2 as id") + expect(env.scope?.unverified).toEqual([ + { path: "models/a.sql", reason: "review failed for this file: EACCES: permission denied" }, + ]) + }) + + test("a lane that throws on one file keeps the other file's findings", async () => { + const env = await runReview({ + changedFiles: two, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ + async check(sql: string) { + if (sql.includes("1")) throw new Error("engine panic") + return { issues: [{ rule: "select-star", message: "avoid", severity: "warning" }], ran: true } + }, + }), + getContent: async (f) => (f.endsWith("a.sql") ? "select 1 as id" : "select 2 as id"), + }) + expect(env.findings.some((f) => f.file === "models/b.sql")).toBe(true) + expect(env.scope?.unverified?.map((u) => u.path)).toEqual(["models/a.sql"]) + }) +}) + +describe("PII abstention", () => { + test("a PII check that could not analyse the model is reported, not treated as clean", async () => { + const env = await runReview({ + changedFiles: [two[0]], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ + async detectPii() { + return { columns: [], abstained: true } + }, + }), + getContent: async () => "select 1 as id", + }) + expect(env.scope?.unverified).toEqual([{ path: "models/a.sql", reason: "the PII check could not analyse this model" }]) + expect(renderSummary(env)).toContain("the PII check could not analyse this model") + }) +}) + +describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => { + function fakeClient(state: { + reviewComments?: Array<{ body: string }> + reviews?: Array<{ id: number; body: string; state: string }> + }) { + const calls: Record = { createReview: [], dismissReview: [], createComment: [], updateComment: [] } + const client = { + async paginate(fn: any) { + if (fn === client.rest.issues.listComments) return [] + if (fn === client.rest.pulls.listReviewComments) return state.reviewComments ?? [] + if (fn === client.rest.pulls.listReviews) return state.reviews ?? [] + return [] + }, + rest: { + issues: { + listComments: () => {}, + async createComment(a: any) { + calls.createComment.push(a) + return { data: { id: 1 } } + }, + async updateComment(a: any) { + calls.updateComment.push(a) + return { data: { id: 1 } } + }, + }, + pulls: { + listReviewComments: () => {}, + listReviews: () => {}, + async get() { + return { data: { head: { sha: "abc" } } } + }, + async createReview(a: any) { + calls.createReview.push(a) + return { data: { id: 99 } } + }, + async dismissReview(a: any) { + calls.dismissReview.push(a) + return { data: {} } + }, + }, + }, + } + return { client: client as unknown as GitHubClient, calls } + } + const target = { token: "t", owner: "o", repo: "r", prNumber: 7 } + const engineFinding = (line?: number) => + makeFinding({ + severity: "warning", + category: "sql_quality", + title: "m: select-star", + body: "avoid", + file: "models/m.sql", + model: "m", + startLine: line, + evidence: { tool: "altimate_core.check", result: {} }, + ruleKey: "q:select-star", + }) + + test("a clean rerun dismisses the earlier blocking altimate review", async () => { + const env = buildEnvelope({ findings: [], tier: "lite", mode: "gate" }) + const { client, calls } = fakeClient({ + reviews: [ + { id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }, + { id: 12, body: "LGTM from a human", state: "CHANGES_REQUESTED" }, + ], + }) + const r = await postGitHubReview(env, target, client) + expect(calls.dismissReview.map((c) => c.review_id)).toEqual([11]) + expect(r.dismissedReviews).toBe(1) + }) + + test("a rerun where the engine or a file failed does not clear the block", async () => { + const blocking = [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }] + const crashed = buildEnvelope({ + findings: [], + tier: "full", + mode: "gate", + engineErrors: [{ method: "check", reason: "native core not loaded", count: 3 }], + }) + const a = fakeClient({ reviews: blocking }) + await postGitHubReview(crashed, target, a.client) + expect(a.calls.dismissReview).toHaveLength(0) + + const fileFailed = buildEnvelope({ + findings: [], + tier: "full", + mode: "gate", + scope: { unverified: [{ path: "models/m.sql", reason: "review failed for this file: boom" }] } as any, + }) + const b = fakeClient({ reviews: blocking }) + await postGitHubReview(fileFailed, target, b.client) + expect(b.calls.dismissReview).toHaveLength(0) + }) + + test("a still-blocking rerun does not dismiss and posts REQUEST_CHANGES", async () => { + const blocking = makeFinding({ + severity: "critical", + category: "lineage_breakage", + title: "m: breaks downstream", + body: "x", + file: "models/m.sql", + evidence: { tool: "impact_analysis", result: {} }, + }) + const env = buildEnvelope({ findings: [blocking], tier: "full", mode: "gate" }) + const { client, calls } = fakeClient({ + reviews: [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }], + }) + await postGitHubReview(env, target, client) + expect(calls.dismissReview).toHaveLength(0) + expect(calls.createReview[0].event).toBe("REQUEST_CHANGES") + }) + + test("findings already on the PR are not re-posted, and with nothing new no review is added", async () => { + const f = engineFinding(3) + const env = buildEnvelope({ findings: [f], tier: "lite", mode: "comment" }) + const { client, calls } = fakeClient({ + reviewComments: [{ body: `⚠️ old\n\n${findingMarker(f.id)}` }], + reviews: [{ id: 5, body: "💬 Reviewed with comments — 1 warning (lite tier)", state: "COMMENTED" }], + }) + const r = await postGitHubReview(env, target, client) + expect(r.inlineSkippedAsPosted).toBe(1) + expect(r.reviewSkipped).toBe(true) + expect(calls.createReview).toHaveLength(0) + expect(calls.createComment.length + calls.updateComment.length).toBe(1) + }) +}) diff --git a/packages/opencode/test/altimate/review-scope-diff.test.ts b/packages/opencode/test/altimate/review-scope-diff.test.ts index 2c11c4bbc4..b4515e22be 100644 --- a/packages/opencode/test/altimate/review-scope-diff.test.ts +++ b/packages/opencode/test/altimate/review-scope-diff.test.ts @@ -337,3 +337,36 @@ describe("reviewPullRequest end to end: monorepo, PR into a non-default branch", expect(env.findings.some((f) => f.file.includes("drift"))).toBe(false) }, 60_000) }) + +describe("a shallow CI checkout without the target branch", () => { + test("the target branch and enough history are fetched to find the merge-base", async () => { + const exec = (await import("node:util")).promisify((await import("node:child_process")).execFile) + const fsp = (await import("node:fs")).promises + const os = await import("node:os") + const pth = await import("node:path") + const g = (cwd: string, ...a: string[]) => exec("git", a, { cwd }).then((r) => r.stdout.trim()) + const origin = await fsp.realpath(await fsp.mkdtemp(pth.join(os.tmpdir(), "shallow-origin-"))) + await g(origin, "init", "-q", "-b", "main") + await g(origin, "config", "user.email", "t@t") + await g(origin, "config", "user.name", "t") + await g(origin, "config", "commit.gpgsign", "false") + await g(origin, "config", "uploadpack.allowReachableSHA1InWant", "true") + for (let i = 0; i < 5; i++) { + await fsp.writeFile(pth.join(origin, `f${i}.sql`), `select ${i}\n`) + await g(origin, "add", "-A") + await g(origin, "commit", "-qm", `main ${i}`) + } + const branchPoint = await g(origin, "rev-parse", "HEAD") + await g(origin, "checkout", "-q", "-b", "feature") + for (let i = 0; i < 3; i++) { + await fsp.writeFile(pth.join(origin, `pr${i}.sql`), `select ${i}\n`) + await g(origin, "add", "-A") + await g(origin, "commit", "-qm", `pr ${i}`) + } + const clone = await fsp.realpath(await fsp.mkdtemp(pth.join(os.tmpdir(), "shallow-clone-"))) + await exec("git", ["clone", "-q", "--depth", "1", "--branch", "feature", `file://${origin}`, clone]) + const resolved = await resolveReviewBase({ cwd: clone, env: { GITHUB_BASE_REF: "main" } }) + expect(resolved.source).toBe("github_env") + expect(resolved.sha).toBe(branchPoint) + }, 60_000) +})