From 49ce896d2afdb60f214ff43582e6effe6473280a Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Thu, 12 Nov 2026 17:06:47 +0530 Subject: [PATCH 1/2] fix(review): failures are never clean: engine errors reported, per-file isolation, PII abstention, gate dismissal - The dispatcher runner records every engine call that throws (method, first reason, count) instead of silently returning empty. They are signed into the envelope (`engineErrors`), named in the comment, counted in `review_run.engine_errors`, and an empty review with engine errors is COMMENT, never APPROVE. - A failure while preparing one file, or a lane throwing for one file, marks that file unverified with the reason; the rest of the PR is still reviewed. PR-level lanes and the core-ran check are guarded the same way. - `detectPii` reports `abstained` when core could not analyse the SQL (check did not run, or its PII parser reported a parse error); the model is listed as not fully verified instead of passing as PII-clean. - When a rerun no longer blocks, earlier blocking altimate reviews are dismissed so the gate does not stay closed. The GitHub client is injectable so dedup, dismissal and skip rules are tested without the network. Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 9 +- .../opencode/src/altimate/review/format.ts | 20 +- .../src/altimate/review/orchestrate.ts | 105 +++++--- .../src/altimate/review/post-github.ts | 43 +++- .../opencode/src/altimate/review/runner.ts | 67 +++-- .../opencode/src/altimate/review/telemetry.ts | 1 + .../opencode/src/altimate/review/verdict.ts | 12 +- .../opencode/src/altimate/telemetry/index.ts | 2 + packages/opencode/src/cli/cmd/review.ts | 3 +- .../altimate/review-failures-gate.test.ts | 237 ++++++++++++++++++ 10 files changed, 438 insertions(+), 61 deletions(-) create mode 100644 packages/opencode/test/altimate/review-failures-gate.test.ts diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index 75fd31edd8..c6bc4b4735 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -311,7 +311,14 @@ rubric: fingerprint, so a push does not repeat findings already on the PR, and a non-blocking rerun with nothing new only updates the summary. - **PII:** flagged only for PII columns the change newly exposes, not for - columns the base already surfaced. + columns the base already surfaced. A model the PII check could not analyse is + listed as not fully verified. +- **Failures are never a clean pass:** if an engine check throws, the comment + names it and the verdict is at most COMMENT, never APPROVE (envelope + `engineErrors`). A failure on one file marks that file unverified and the rest + of the PR is still reviewed. +- **The gate clears itself:** when a rerun no longer blocks, the earlier blocking + altimate review is dismissed (needs `pull-requests: write`). ### Risk-tier path tokens diff --git a/packages/opencode/src/altimate/review/format.ts b/packages/opencode/src/altimate/review/format.ts index 8f6cf489dd..0ae0c40acb 100644 --- a/packages/opencode/src/altimate/review/format.ts +++ b/packages/opencode/src/altimate/review/format.ts @@ -206,13 +206,19 @@ export function coverageNotes(env: VerdictEnvelope): string[] { const undecided = env.findings.filter((f) => f.degraded).length if (undecided) notes.push(`${undecided} finding(s) could not be decided by the engine and are marked _unverified_.`) const unverified = env.scope?.unverified ?? [] - if (unverified.length) { - const head = unverified.slice(0, 5).map((u) => `\`${u.path}\``).join(", ") - const more = unverified.length > 5 ? ` and ${unverified.length - 5} more` : "" - notes.push( - `${unverified.length} model(s) not verified by the engine — ${unverified[0].reason}: ${head}${more}. ` + - "Run `dbt compile` (and compile the base into `target-base/`) for equivalence and lineage checks.", - ) + const byReason = new Map() + for (const u of unverified) byReason.set(u.reason, [...(byReason.get(u.reason) ?? []), u.path]) + for (const [reason, paths] of byReason) { + const head = paths.slice(0, 5).map((p) => `\`${p}\``).join(", ") + const more = paths.length > 5 ? ` and ${paths.length - 5} more` : "" + const hint = /compiled SQL/.test(reason) + ? " Run `dbt compile` (and compile the base into `target-base/`) for equivalence and lineage checks." + : "" + notes.push(`${paths.length} model(s) not fully verified — ${reason}: ${head}${more}.${hint}`) + } + if (env.engineErrors?.length) { + const list = env.engineErrors.map((e) => `${e.method} ×${e.count} (${e.reason})`).join("; ") + notes.push(`Some engine checks failed and did not run: ${list}. This verdict is not a clean approval.`) } if (env.scope?.outOfScopeFiles) notes.push(`${env.scope.outOfScopeFiles} changed file(s) outside the dbt project were not reviewed.`) diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 643541af87..05b7f5f306 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -101,6 +101,8 @@ export interface GradeResult { export interface CheckResult { issues?: Array<{ rule: string; message: string; line?: number; severity?: string; category?: string }> piiColumns?: string[] + /** Core's PII pass could not parse the SQL: `piiColumns` is unknown, not empty. */ + piiAbstained?: boolean /** * True when the native core lint actually ran (parsed the SQL via the AST), * even if it found zero issues. Distinguishes "clean SQL" from "core @@ -118,7 +120,10 @@ export interface ReviewRunner { grade(sql: string, dialect: string): Promise check(sql: string, dialect: string, baseSql?: string): Promise equivalence(oldSql: string, newSql: string, dialect?: string): Promise - detectPii(sql: string, dialect: string): Promise<{ columns: string[] }> + /** `abstained`: the PII pass could not analyse the SQL, so `columns` is unknown. */ + detectPii(sql: string, dialect: string): Promise<{ columns: string[]; abstained?: boolean }> + /** Engine calls that threw during this review (method, first reason, count). */ + failures?(): Array<{ method: string; reason: string; count: number }> /** * Lexical scan (reserved-word aliases + dialect operators) — the curated lists * and detection live in the compiled core; this passes raw added diff lines. @@ -1147,35 +1152,55 @@ export async function runReview(input: OrchestrateInput): Promise false) } + // PR-level lanes (across files) that threw: reported with the engine errors. + const laneErrors: Array<{ method: string; reason: string; count: number }> = [] + const prLevel = (method: string, lane: Promise) => + lane.catch((err) => { + laneErrors.push({ method, reason: (err instanceof Error ? err.message : String(err)).slice(0, 200), count: 1 }) + return [] as Finding[] + }) + const failedFile = (file: { path: string }, err: unknown) => + unverified.push({ + path: file.path, + reason: `review failed for this file: ${(err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 160)}`, + }) await Promise.all( modelFiles.map(async (file) => { - const model = modelNameFromPath(file.path) - // For renames the previous content lives at oldPath, not the new path. - const oldRef = file.oldPath ?? file.path - const withOld = hasOldSide(file) - const [newSql, oldSql, compiledNew, compiledOld] = await Promise.all([ - file.status !== "deleted" ? getContent?.(file.path, "new") : Promise.resolve(undefined), - withOld ? getContent?.(oldRef, "old") : Promise.resolve(undefined), - file.status !== "deleted" ? getCompiled?.(file.path, "new") : Promise.resolve(undefined), - withOld ? getCompiled?.(oldRef, "old") : Promise.resolve(undefined), - ]) - // Engine lanes need dbt-compiled SQL. Raw SQL is used only when it holds - // no Jinja: the AST engine reads `{{ config() }}` as an injection attempt - // and a mixed raw-vs-compiled pair can never be proven equivalent, so a - // Jinja model without compiled output is marked unverified instead of - // producing warnings about its template syntax. The dbt-patterns lane - // always uses raw (it needs the Jinja). - const engineNewSql = compiledNew ?? (file.kind === "model_sql" && hasJinja(newSql) ? undefined : newSql) - const engineOldSql = compiledOld ?? (file.kind === "model_sql" && hasJinja(oldSql) ? undefined : oldSql) - if (file.status !== "deleted" && newSql && !engineNewSql) - unverified.push({ path: file.path, reason: "no compiled SQL for the head (run dbt compile)" }) - else if (withOld && oldSql && !engineOldSql) - unverified.push({ path: file.path, reason: "no compiled SQL for the base (compile the base into target-base)" }) - const impact = await input.runner.impact(model) - if (impact.hasManifest) anyManifest = true - const pii = engineNewSql ? (await input.runner.detectPii(engineNewSql, dialect)).columns : [] - const complex = engineNewSql && input.runner.isComplex ? await input.runner.isComplex(engineNewSql) : false - ctxByPath.set(file.path, { file, impact, pii, newSql, oldSql, engineNewSql, engineOldSql, complex }) + try { + const model = modelNameFromPath(file.path) + // For renames the previous content lives at oldPath, not the new path. + const oldRef = file.oldPath ?? file.path + const withOld = hasOldSide(file) + const [newSql, oldSql, compiledNew, compiledOld] = await Promise.all([ + file.status !== "deleted" ? getContent?.(file.path, "new") : Promise.resolve(undefined), + withOld ? getContent?.(oldRef, "old") : Promise.resolve(undefined), + file.status !== "deleted" ? getCompiled?.(file.path, "new") : Promise.resolve(undefined), + withOld ? getCompiled?.(oldRef, "old") : Promise.resolve(undefined), + ]) + // Engine lanes need dbt-compiled SQL. Raw SQL is used only when it holds + // no Jinja: the AST engine reads `{{ config() }}` as an injection attempt + // and a mixed raw-vs-compiled pair can never be proven equivalent, so a + // Jinja model without compiled output is marked unverified instead of + // producing warnings about its template syntax. The dbt-patterns lane + // always uses raw (it needs the Jinja). + const engineNewSql = compiledNew ?? (file.kind === "model_sql" && hasJinja(newSql) ? undefined : newSql) + const engineOldSql = compiledOld ?? (file.kind === "model_sql" && hasJinja(oldSql) ? undefined : oldSql) + if (file.status !== "deleted" && newSql && !engineNewSql) + unverified.push({ path: file.path, reason: "no compiled SQL for the head (run dbt compile)" }) + else if (withOld && oldSql && !engineOldSql) + unverified.push({ path: file.path, reason: "no compiled SQL for the base (compile the base into target-base)" }) + const impact = await input.runner.impact(model) + if (impact.hasManifest) anyManifest = true + const piiRes = engineNewSql ? await input.runner.detectPii(engineNewSql, dialect) : { columns: [] } + if (piiRes.abstained) unverified.push({ path: file.path, reason: "the PII check could not analyse this model" }) + const pii = piiRes.columns + const complex = engineNewSql && input.runner.isComplex ? await input.runner.isComplex(engineNewSql) : false + ctxByPath.set(file.path, { file, impact, pii, newSql, oldSql, engineNewSql, engineOldSql, complex }) + } catch (err) { + // One file's failure (unreadable content, an engine crash) degrades that + // file only; it must not abort the whole review. + failedFile(file, err) + } }), ) @@ -1327,20 +1352,32 @@ export async function runReview(input: OrchestrateInput): Promise + t.catch((err) => { + if (!laneFailed) failedFile(ctx.file, err) + laneFailed = true + return [] as Finding[] + }), + ), + ) + all.push(...settled) } // PR-level: cross-model sibling filter-consistency (divergent filters on a // shared upstream across the changed models). if (lanes.has("semantic_change")) { - all.push(await siblingConsistencyLane([...ctxByPath.values()], input.runner)) + all.push(await prLevel("sibling_consistency", siblingConsistencyLane([...ctxByPath.values()], input.runner))) } // PR-level: topology-aware equivalence for merge refactors (N models deleted + // 1 added that supersedes them). Per-model equivalence can't see these because // the new model has no 1:1 base. Advisory only (capped at warning). if (lanes.has("semantic_change")) { - all.push(await topologyEquivalenceLane(modelFiles, input.runner, getCompiled, dialect)) + all.push(await prLevel("topology_equivalence", topologyEquivalenceLane(modelFiles, input.runner, getCompiled, dialect))) } // schema.yml-level detectors (test removal) — run on changed YAML files @@ -1393,7 +1430,9 @@ export async function runReview(input: OrchestrateInput): Promise { if (!ctx.engineNewSql || ctx.file.status === "deleted") return // Cache hit: qualityLane already ran this exact check. - const cr = await input.runner.check(ctx.engineNewSql, dialect, ctx.engineOldSql) + const cr = await input.runner + .check(ctx.engineNewSql, dialect, ctx.engineOldSql) + .catch(() => ({ ran: false }) as CheckResult) if (cr.ran) coreRanFiles.add(ctx.file.path) }), ) @@ -1536,7 +1575,9 @@ export async function runReview(input: OrchestrateInput): Promise { - const octo = new Octokit({ auth: target.token }) +/** The Octokit surface the poster uses; injectable so the posting rules + * (dedup, dismissal, skip) are testable without the network. */ +export type GitHubClient = Pick & { + rest: Pick +} + +export async function postGitHubReview( + env: VerdictEnvelope, + target: GitHubTarget, + client?: GitHubClient, +): Promise { + const octo = client ?? new Octokit({ auth: target.token }) const { owner, repo, prNumber } = target const result: PostResult = { inlineFellBack: false } @@ -99,13 +111,16 @@ export async function postGitHubReview(env: VerdictEnvelope, target: GitHubTarge // ignores line numbers) is not posted again on the next push. let posted = new Set() let priorBotReview = false + let staleBlocking: number[] = [] try { const [reviewComments, reviews] = await Promise.all([ octo.paginate(octo.rest.pulls.listReviewComments, { owner, repo, pull_number: prNumber, per_page: 100 }), octo.paginate(octo.rest.pulls.listReviews, { owner, repo, pull_number: prNumber, per_page: 100 }), ]) posted = postedFindingIds(reviewComments.map((c) => c.body)) - priorBotReview = reviews.some((r) => BOT_REVIEW_RE.test(r.body ?? "")) + const botReviews = reviews.filter((r) => BOT_REVIEW_RE.test(r.body ?? "")) + priorBotReview = botReviews.length > 0 + staleBlocking = botReviews.filter((r) => r.state === "CHANGES_REQUESTED").map((r) => r.id) } catch { /* listing failed: post everything rather than nothing */ } @@ -113,6 +128,28 @@ export async function postGitHubReview(env: VerdictEnvelope, target: GitHubTarge const withLine = env.findings.filter((f) => typeof f.startLine === "number").length result.inlineSkippedAsPosted = withLine - comments.length + // Gate lifecycle: when this run no longer blocks, an earlier blocking + // altimate review would keep the PR blocked forever. Dismiss it, pointing at + // the new verdict. (Requires `pull-requests: write`; a failure is non-fatal.) + if (event === "COMMENT" && staleBlocking.length) { + let dismissed = 0 + for (const id of staleBlocking) { + try { + await octo.rest.pulls.dismissReview({ + owner, + repo, + pull_number: prNumber, + review_id: id, + message: `Superseded by a newer altimate review: ${reviewBody}`, + }) + dismissed++ + } catch { + /* keep going; the summary still shows the current verdict */ + } + } + result.dismissedReviews = dismissed + } + // Nothing new inline, not blocking, and the PR already has an altimate // review: the updated summary comment says it all; another "Reviewed with // comments" entry per push is noise. A blocking verdict always posts. diff --git a/packages/opencode/src/altimate/review/runner.ts b/packages/opencode/src/altimate/review/runner.ts index eec3732d81..aa59b398ca 100644 --- a/packages/opencode/src/altimate/review/runner.ts +++ b/packages/opencode/src/altimate/review/runner.ts @@ -102,6 +102,16 @@ export interface DispatcherRunnerOptions { export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRunner { const checkCache = new Map() + // Engine calls that threw, by method → first reason. Each method still + // returns its empty result so one failure never aborts the review, but the + // failures are reported (envelope `engineErrors`, the comment, telemetry) + // instead of reading as "no findings". + const failures = new Map() + function fail(method: string, err: unknown) { + const reason = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 200) + const prev = failures.get(method) + failures.set(method, { reason: prev?.reason ?? reason, count: (prev?.count ?? 0) + 1 }) + } let manifestPromise: Promise | undefined async function loadManifest(): Promise { @@ -252,8 +262,14 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun ].filter((c): c is string => typeof c === "string" && !!c) // ran=true: the core parsed and analyzed the SQL (even if zero issues). // This lets the orchestrator defer structural checks to the AST lint. - out = { issues, piiColumns: [...new Set(piiColumns)], ran: true } - } catch { + out = { + issues, + piiColumns: [...new Set(piiColumns)], + ran: true, + piiAbstained: !!(data.pii as any)?.parse_error, + } + } catch (err) { + fail("check", err) out = { issues: [], piiColumns: [], ran: false } } checkCache.set(cacheKey, out) @@ -310,7 +326,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.grade", { sql, schema_context: await resolveSchema() }) const data = (res.data ?? {}) as Record return { grade: data.overall_grade ?? data.grade } - } catch { + } catch (err) { + fail("grade", err) return {} } }, @@ -324,7 +341,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.grain", { sql }) const d = (res.data ?? {}) as Record return { group_by: asArray(d.group_by), dedup_partition: asArray(d.dedup_partition) } - } catch { + } catch (err) { + fail("grain", err) return { group_by: [], dedup_partition: [] } } }, @@ -337,14 +355,16 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun try { const res = await Dispatcher.call("altimate_core.dbt_config_lint", { sql: rawSql }) out.push(...asArray((res.data as any)?.findings)) - } catch { + } catch (err) { + fail("dbtConfigLint", err) /* skip */ } if (oldRawSql) { try { const res = await Dispatcher.call("altimate_core.dbt_config_diff", { base_sql: oldRawSql, head_sql: rawSql }) out.push(...asArray((res.data as any)?.findings)) - } catch { + } catch (err) { + fail("dbtConfigLint", err) /* skip */ } } @@ -358,7 +378,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun try { const res = await Dispatcher.call("altimate_core.structural_diff", { base_sql: baseSql, head_sql: headSql }) return asArray((res.data as any)?.findings) - } catch { + } catch (err) { + fail("structuralDiff", err) return [] } }, @@ -368,7 +389,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.source_filters", { sql }) const d = (res.data ?? {}) as Record return (d.filters ?? {}) as Record - } catch { + } catch (err) { + fail("sourceFilters", err) return {} } }, @@ -397,7 +419,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.metadata", { sql }) const d = (res.data ?? {}) as Record return !!(d.has_window_functions || d.has_subqueries || (typeof d.node_count === "number" && d.node_count >= 12)) - } catch { + } catch (err) { + fail("isComplex", err) return false } }, @@ -423,7 +446,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun confidence: typeof c.confidence === "number" ? c.confidence : 0, masking: c.suggested_masking ?? undefined, })) - } catch { + } catch (err) { + fail("classifyPii", err) return [] } }, @@ -450,7 +474,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun rowsChanged: o.rows_changed ?? o.different ?? o.changed, summary: o.summary ?? r.summary, } - } catch { + } catch (err) { + fail("dataDiff", err) return null } }, @@ -468,7 +493,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun source: String(e.source ?? ""), target: String(e.target ?? ""), })) - } catch { + } catch (err) { + fail("columnLineage", err) return [] } }, @@ -484,7 +510,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun message: String(f.message ?? ""), line: f.line, })) - } catch { + } catch (err) { + fail("lexicalScan", err) return [] } }, @@ -529,16 +556,24 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun differences: diffs.map((d) => d?.description ?? String(d)), confidence: EngineCoerce.bandConfidence(data.confidence), } - } catch { + } catch (err) { + fail("equivalence", err) return { decided: false } } }, - async detectPii(sql: string): Promise<{ columns: string[] }> { + async detectPii(sql: string): Promise<{ columns: string[]; abstained?: boolean }> { // Text-based PII comes from altimate_core.check (schema.detect_pii is // warehouse-based and unavailable in CI). Reuses the memoized check. + // When core could not analyse the SQL (check did not run, or its PII + // parser reported a parse error) the empty list means "unknown", not + // "no PII": say so. const result = await runCheck(sql) - return { columns: result.piiColumns ?? [] } + return { columns: result.piiColumns ?? [], abstained: !result.ran || result.piiAbstained === true } + }, + + failures() { + return [...failures].map(([method, f]) => ({ method, reason: f.reason, count: f.count })) }, } } diff --git a/packages/opencode/src/altimate/review/telemetry.ts b/packages/opencode/src/altimate/review/telemetry.ts index 3fbe4af1bb..0e6f4c0ad5 100644 --- a/packages/opencode/src/altimate/review/telemetry.ts +++ b/packages/opencode/src/altimate/review/telemetry.ts @@ -117,6 +117,7 @@ export function emitReviewRun(input: { ai_duration_ms: env.ai?.durationMs, base_source: env.scope?.baseSource, out_of_scope_files: env.scope?.outOfScopeFiles ?? 0, + engine_errors: env.engineErrors?.reduce((n, e) => n + e.count, 0) ?? 0, stale_manifest: env.staleManifest === true, critical: env.summary.critical, warning: env.summary.warning, diff --git a/packages/opencode/src/altimate/review/verdict.ts b/packages/opencode/src/altimate/review/verdict.ts index 595d8087df..c8f510dbb9 100644 --- a/packages/opencode/src/altimate/review/verdict.ts +++ b/packages/opencode/src/altimate/review/verdict.ts @@ -154,6 +154,11 @@ export const VerdictEnvelope = z.object({ staleManifest: z.boolean().optional(), /** Base resolution and review coverage (see ReviewScope). */ scope: ReviewScope.optional(), + /** Engine calls that failed during the review. Their checks did not run, so + * the verdict is never a clean APPROVE when this is present. */ + engineErrors: z + .array(z.object({ method: z.string(), reason: z.string(), count: z.number().int().positive() })) + .optional(), /** The review policy applied: from the base branch, the working tree, or * defaults; its hash; and settings the PR changes but that were not applied. */ policy: z @@ -233,6 +238,7 @@ export interface BuildEnvelopeInput { scope?: ReviewScope ai?: VerdictEnvelope["ai"] policy?: VerdictEnvelope["policy"] + engineErrors?: VerdictEnvelope["engineErrors"] } function summarize(findings: Finding[], degraded: boolean): VerdictEnvelope["summary"] { @@ -244,7 +250,10 @@ function summarize(findings: Finding[], degraded: boolean): VerdictEnvelope["sum /** Assemble the verdict envelope (unsigned). Call signEnvelope to sign it. */ export function buildEnvelope(input: BuildEnvelopeInput): VerdictEnvelope { const rubric = input.rubric ?? DEFAULT_RUBRIC - const ideal = computeIdealVerdict(input.findings, rubric) + // An engine failure means some checks did not run: "no findings" is then + // unknown, not clean, so the verdict cannot be APPROVE. + const computed = computeIdealVerdict(input.findings, rubric) + const ideal = computed === "APPROVE" && input.engineErrors?.length ? "COMMENT" : computed const verdict = applyMode(ideal, input.mode) const degraded = input.degraded ?? input.findings.some((f) => f.degraded) return VerdictEnvelope.parse({ @@ -264,6 +273,7 @@ export function buildEnvelope(input: BuildEnvelopeInput): VerdictEnvelope { scope: input.scope, ai: input.ai, policy: input.policy, + engineErrors: input.engineErrors, generatedAt: input.generatedAt, }) } diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 8566507850..288b69319a 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -1608,6 +1608,8 @@ export namespace Telemetry { /** Where the review base came from (flag, github_event, default_branch, …). */ base_source?: string out_of_scope_files?: number + /** Engine calls that threw during the review (their checks did not run). */ + engine_errors?: number stale_manifest?: boolean critical?: number warning?: number diff --git a/packages/opencode/src/cli/cmd/review.ts b/packages/opencode/src/cli/cmd/review.ts index cef6942c20..b1677b7fe1 100644 --- a/packages/opencode/src/cli/cmd/review.ts +++ b/packages/opencode/src/cli/cmd/review.ts @@ -194,7 +194,8 @@ export const ReviewCommand = cmd({ `Posted review to ${where}` + (r.inlineFellBack ? " (inline comments fell back to summary-only)" : "") + (r.inlineSkippedAsPosted ? ` (${r.inlineSkippedAsPosted} inline already on the PR)` : "") + - (r.reviewSkipped ? " (summary updated; no new review needed)" : ""), + (r.reviewSkipped ? " (summary updated; no new review needed)" : "") + + (r.dismissedReviews ? ` (dismissed ${r.dismissedReviews} earlier blocking review(s))` : ""), ) } } diff --git a/packages/opencode/test/altimate/review-failures-gate.test.ts b/packages/opencode/test/altimate/review-failures-gate.test.ts new file mode 100644 index 0000000000..07f2c4e252 --- /dev/null +++ b/packages/opencode/test/altimate/review-failures-gate.test.ts @@ -0,0 +1,237 @@ +import { describe, test, expect } from "bun:test" +import { runReview, type ReviewRunner } from "../../src/altimate/review/orchestrate" +import { DEFAULT_REVIEW_CONFIG } from "../../src/altimate/review/config" +import { DEFAULT_RUBRIC } from "../../src/altimate/review/rubric" +import { makeFinding } from "../../src/altimate/review/finding" +import { buildEnvelope, signEnvelope, verifyEnvelope } from "../../src/altimate/review/verdict" +import { renderSummary, findingMarker } from "../../src/altimate/review/format" +import { postGitHubReview, type GitHubClient } from "../../src/altimate/review/post-github" + +/** + * A review must never look clean because something broke: engine failures and + * PII abstentions are reported, one file's crash degrades only that file, and + * a blocking review is cleared when a rerun no longer blocks. + */ + +function runner(over: Partial = {}): ReviewRunner { + return { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return { grade: "B" } + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: true, equivalent: true } + }, + async detectPii() { + return { columns: [] } + }, + ...over, + } as ReviewRunner +} + +const two = [ + { path: "models/a.sql", status: "added" as const, diff: "+select 1 as id\n" }, + { path: "models/b.sql", status: "added" as const, diff: "+select 2 as id\n" }, +] + +describe("engine failures are never clean", () => { + test("no findings plus an engine failure is COMMENT, not APPROVE, and the failure is shown", async () => { + const env = await runReview({ + changedFiles: two, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ failures: () => [{ method: "check", reason: "native core not loaded", count: 2 }] }), + getContent: async (f) => (f.endsWith("a.sql") ? "select 1 as id" : "select 2 as id"), + }) + expect(env.findings).toHaveLength(0) + expect(env.idealVerdict).toBe("COMMENT") + expect(env.engineErrors).toEqual([{ method: "check", reason: "native core not loaded", count: 2 }]) + expect(renderSummary(env)).toContain("Some engine checks failed and did not run: check ×2 (native core not loaded)") + }) + + test("engine errors are signed", () => { + const env = signEnvelope( + buildEnvelope({ findings: [], tier: "lite", mode: "comment", engineErrors: [{ method: "equivalence", reason: "x", count: 1 }] }), + ) + expect(env.verdict).toBe("COMMENT") + expect(verifyEnvelope(env)).toBe(true) + expect(verifyEnvelope({ ...env, engineErrors: undefined })).toBe(false) + }) + + test("without failures an empty review still approves", () => { + expect(buildEnvelope({ findings: [], tier: "lite", mode: "comment" }).verdict).toBe("APPROVE") + }) +}) + +describe("one file's failure degrades only that file", () => { + test("unreadable content for one model: the other is still reviewed", async () => { + let checked: string[] = [] + const env = await runReview({ + changedFiles: two, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ + async check(sql: string) { + checked.push(sql) + return { issues: [], ran: true } + }, + }), + getContent: async (f) => { + if (f.endsWith("a.sql")) throw new Error("EACCES: permission denied") + return "select 2 as id" + }, + }) + expect(checked).toContain("select 2 as id") + expect(env.scope?.unverified).toEqual([ + { path: "models/a.sql", reason: "review failed for this file: EACCES: permission denied" }, + ]) + }) + + test("a lane that throws on one file keeps the other file's findings", async () => { + const env = await runReview({ + changedFiles: two, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ + async check(sql: string) { + if (sql.includes("1")) throw new Error("engine panic") + return { issues: [{ rule: "select-star", message: "avoid", severity: "warning" }], ran: true } + }, + }), + getContent: async (f) => (f.endsWith("a.sql") ? "select 1 as id" : "select 2 as id"), + }) + expect(env.findings.some((f) => f.file === "models/b.sql")).toBe(true) + expect(env.scope?.unverified?.map((u) => u.path)).toEqual(["models/a.sql"]) + }) +}) + +describe("PII abstention", () => { + test("a PII check that could not analyse the model is reported, not treated as clean", async () => { + const env = await runReview({ + changedFiles: [two[0]], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ + async detectPii() { + return { columns: [], abstained: true } + }, + }), + getContent: async () => "select 1 as id", + }) + expect(env.scope?.unverified).toEqual([{ path: "models/a.sql", reason: "the PII check could not analyse this model" }]) + expect(renderSummary(env)).toContain("the PII check could not analyse this model") + }) +}) + +describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => { + function fakeClient(state: { + reviewComments?: Array<{ body: string }> + reviews?: Array<{ id: number; body: string; state: string }> + }) { + const calls: Record = { createReview: [], dismissReview: [], createComment: [], updateComment: [] } + const client = { + async paginate(fn: any) { + if (fn === client.rest.issues.listComments) return [] + if (fn === client.rest.pulls.listReviewComments) return state.reviewComments ?? [] + if (fn === client.rest.pulls.listReviews) return state.reviews ?? [] + return [] + }, + rest: { + issues: { + listComments: () => {}, + async createComment(a: any) { + calls.createComment.push(a) + return { data: { id: 1 } } + }, + async updateComment(a: any) { + calls.updateComment.push(a) + return { data: { id: 1 } } + }, + }, + pulls: { + listReviewComments: () => {}, + listReviews: () => {}, + async get() { + return { data: { head: { sha: "abc" } } } + }, + async createReview(a: any) { + calls.createReview.push(a) + return { data: { id: 99 } } + }, + async dismissReview(a: any) { + calls.dismissReview.push(a) + return { data: {} } + }, + }, + }, + } + return { client: client as unknown as GitHubClient, calls } + } + const target = { token: "t", owner: "o", repo: "r", prNumber: 7 } + const engineFinding = (line?: number) => + makeFinding({ + severity: "warning", + category: "sql_quality", + title: "m: select-star", + body: "avoid", + file: "models/m.sql", + model: "m", + startLine: line, + evidence: { tool: "altimate_core.check", result: {} }, + ruleKey: "q:select-star", + }) + + test("a clean rerun dismisses the earlier blocking altimate review", async () => { + const env = buildEnvelope({ findings: [], tier: "lite", mode: "gate" }) + const { client, calls } = fakeClient({ + reviews: [ + { id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }, + { id: 12, body: "LGTM from a human", state: "CHANGES_REQUESTED" }, + ], + }) + const r = await postGitHubReview(env, target, client) + expect(calls.dismissReview.map((c) => c.review_id)).toEqual([11]) + expect(r.dismissedReviews).toBe(1) + }) + + test("a still-blocking rerun does not dismiss and posts REQUEST_CHANGES", async () => { + const blocking = makeFinding({ + severity: "critical", + category: "lineage_breakage", + title: "m: breaks downstream", + body: "x", + file: "models/m.sql", + evidence: { tool: "impact_analysis", result: {} }, + }) + const env = buildEnvelope({ findings: [blocking], tier: "full", mode: "gate" }) + const { client, calls } = fakeClient({ + reviews: [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }], + }) + await postGitHubReview(env, target, client) + expect(calls.dismissReview).toHaveLength(0) + expect(calls.createReview[0].event).toBe("REQUEST_CHANGES") + }) + + test("findings already on the PR are not re-posted, and with nothing new no review is added", async () => { + const f = engineFinding(3) + const env = buildEnvelope({ findings: [f], tier: "lite", mode: "comment" }) + const { client, calls } = fakeClient({ + reviewComments: [{ body: `⚠️ old\n\n${findingMarker(f.id)}` }], + reviews: [{ id: 5, body: "💬 Reviewed with comments — 1 warning (lite tier)", state: "COMMENTED" }], + }) + const r = await postGitHubReview(env, target, client) + expect(r.inlineSkippedAsPosted).toBe(1) + expect(r.reviewSkipped).toBe(true) + expect(calls.createReview).toHaveLength(0) + expect(calls.createComment.length + calls.updateComment.length).toBe(1) + }) +}) From 6736c7c1c9f82eb0a94aead65c60c3aa01baf2f6 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 13 Nov 2026 11:10:41 +0530 Subject: [PATCH 2/2] fix(review): a failed run never clears a block; shallow CI checkouts fetch the base; upgrade note for gate users - a stale `CHANGES_REQUESTED` review is dismissed only when the new run completed (no `engineErrors`, no file that failed); an engine crash no longer unblocks a PR by making its findings disappear - in CI, a missing target branch is fetched from `origin` and a shallow history deepened (50/200/1000) until the merge-base exists, so the default `actions/checkout` works without `fetch-depth: 0`; still an error when that fails - docs: what changes for existing `gate` setups (pattern rules no longer block unless listed in `rubric.blockingRules`) and the new checkout behavior Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 10 ++++- packages/opencode/src/altimate/review/git.ts | 38 +++++++++++++++++-- .../src/altimate/review/post-github.ts | 10 ++++- .../altimate/review-failures-gate.test.ts | 23 +++++++++++ .../test/altimate/review-scope-diff.test.ts | 33 ++++++++++++++++ 5 files changed, 108 insertions(+), 6 deletions(-) diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index c6bc4b4735..78e5df7115 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -108,7 +108,7 @@ Options: | Flag | Description | |------|-------------| -| `--base ` | Base git ref. Defaults to the pull request's target branch in CI (GitHub `pull_request.base.ref` or `GITHUB_BASE_REF`, GitLab `CI_MERGE_REQUEST_TARGET_BRANCH_NAME`), else the remote default branch. Always reduced to the merge-base with the head, so commits the base gained after the PR branched are never reviewed. In CI, a target branch that is not fetched is an error (use `fetch-depth: 0`). | +| `--base ` | Base git ref. Defaults to the pull request's target branch in CI (GitHub `pull_request.base.ref` or `GITHUB_BASE_REF`, GitLab `CI_MERGE_REQUEST_TARGET_BRANCH_NAME`), else the remote default branch. Always reduced to the merge-base with the head, so commits the base gained after the PR branched are never reviewed. In CI, a shallow checkout is handled: the target branch is fetched and the history deepened until the merge-base is found. If that fails (no `origin`, no access), the review stops with an error rather than diffing against another branch. | | `--head ` | Head git ref. Omit to review the working tree. | | `--manifest ` | Path to the compiled `manifest.json`. When omitted, the reviewer walks up from the current directory to find the nearest `dbt_project.yml` and uses its adjacent `target/manifest.json`; the discovered path is logged to stderr. | | `--mode comment\|gate` | `comment` never blocks; `gate` exits non-zero on `REQUEST_CHANGES`. | @@ -203,7 +203,7 @@ jobs: | Input | How to produce it | Unlocks | |---|---|---| -| Full git history of the PR's target branch | `actions/checkout` with `fetch-depth: 0` | Correct base: the diff starts at the merge-base with the PR's target branch. A target branch that is not fetched is an error, never a silent fallback to `main`. | +| The PR's target branch and the merge-base | Fetched automatically from `origin` when the checkout is shallow; `fetch-depth: 0` avoids the extra fetches | Correct base: the diff starts at the merge-base with the PR's target branch. When it can't be fetched, the review stops with an error, never a silent fallback to `main`. | | `target/manifest.json` | `dbt compile` (or `dbt parse` for lint + lineage only) | Blast radius, lineage breakage, tiering. Without it the run is lint-only. | | `target/compiled/…` | `dbt compile` | Engine checks on the head (lint, grade, PII, grain). | | `target-base/compiled/…` | Compile the base ref into `target-base/` (below) | Equivalence proofs and column-breakage for modified models. | @@ -296,6 +296,12 @@ rubric: rules) or `ai`. Only `engine` findings count toward REQUEST_CHANGES. A pattern rule you trust can be allowed to block by naming its id in `rubric.blockingRules`. The advisory AI lane never blocks. +- **Upgrading a `gate` setup.** Before this policy, any critical finding in a + `blockOn` category could block, including pattern rules such as the regex + PII check. Now pattern-rule findings comment but don't block. To keep a + pattern rule blocking, add its id to `rubric.blockingRules`. A blocking + review is dismissed only by a later run that completed: when an engine call + or a file failed, the earlier block stays. ### What the PR comment shows diff --git a/packages/opencode/src/altimate/review/git.ts b/packages/opencode/src/altimate/review/git.ts index 2455ea65f9..5dd6ee7f65 100644 --- a/packages/opencode/src/altimate/review/git.ts +++ b/packages/opencode/src/altimate/review/git.ts @@ -191,6 +191,33 @@ async function mergeBase(a: string, b: string, cwd: string): Promise Promise): Promise { + if (branch.startsWith("-")) return false + try { + await git(["fetch", "--no-tags", "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`], cwd) + } catch { + return false + } + if (await found()) return true + for (const depth of [50, 200, 1000]) { + try { + if ((await git(["rev-parse", "--is-shallow-repository"], cwd)).trim() !== "true") return false + const headSha = (await git(["rev-parse", "HEAD"], cwd)).trim() + await git(["fetch", "--no-tags", `--deepen=${depth}`, "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`, headSha], cwd) + } catch { + return false + } + if (await found()) return true + } + return false +} + /** The PR's target branch from the CI provider, when the run is for a PR/MR. */ async function ciTargetBranch(env: NodeJS.ProcessEnv): Promise<{ branch: string; source: BaseSource } | undefined> { if (env.GITHUB_EVENT_PATH) { @@ -230,10 +257,15 @@ export async function resolveReviewBase(opts: ResolveBaseOptions): Promise { + for (const ref of [`origin/${target.branch}`, target.branch]) { + const sha = await mergeBase(ref, head, opts.cwd) + if (sha) return { ref, sha, source: target.source } + } + return undefined } + const found = (await find()) ?? ((await fetchTargetHistory(target.branch, opts.cwd, find)) ? await find() : undefined) + if (found) return found throw new Error( `PR target branch '${target.branch}' (${target.source}) is not available locally. ` + `Fetch it (actions/checkout with fetch-depth: 0, or git fetch origin ${target.branch}) or pass --base.`, diff --git a/packages/opencode/src/altimate/review/post-github.ts b/packages/opencode/src/altimate/review/post-github.ts index bca5bb59c3..9bb445a773 100644 --- a/packages/opencode/src/altimate/review/post-github.ts +++ b/packages/opencode/src/altimate/review/post-github.ts @@ -64,6 +64,12 @@ export interface PostResult { } /** Headline of a review this bot posted (see verdictHeadline). */ +/** True when no engine call and no file failed, so missing findings mean "fixed", not "not checked". */ +export function runCompleted(env: VerdictEnvelope): boolean { + if (env.engineErrors?.length) return false + return !(env.scope?.unverified ?? []).some((u) => u.reason.startsWith("review failed for this file")) +} + const BOT_REVIEW_RE = /(Approved|Reviewed with comments|Changes requested)\b.*\((trivial|lite|full) tier/ /** The Octokit surface the poster uses; injectable so the posting rules @@ -131,7 +137,9 @@ export async function postGitHubReview( // Gate lifecycle: when this run no longer blocks, an earlier blocking // altimate review would keep the PR blocked forever. Dismiss it, pointing at // the new verdict. (Requires `pull-requests: write`; a failure is non-fatal.) - if (event === "COMMENT" && staleBlocking.length) { + // Only a run that actually completed may clear a block: when an engine call + // or a file failed, the blocking finding may simply not have been produced. + if (event === "COMMENT" && staleBlocking.length && runCompleted(env)) { let dismissed = 0 for (const id of staleBlocking) { try { diff --git a/packages/opencode/test/altimate/review-failures-gate.test.ts b/packages/opencode/test/altimate/review-failures-gate.test.ts index 07f2c4e252..3fcc45fc79 100644 --- a/packages/opencode/test/altimate/review-failures-gate.test.ts +++ b/packages/opencode/test/altimate/review-failures-gate.test.ts @@ -203,6 +203,29 @@ describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => expect(r.dismissedReviews).toBe(1) }) + test("a rerun where the engine or a file failed does not clear the block", async () => { + const blocking = [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }] + const crashed = buildEnvelope({ + findings: [], + tier: "full", + mode: "gate", + engineErrors: [{ method: "check", reason: "native core not loaded", count: 3 }], + }) + const a = fakeClient({ reviews: blocking }) + await postGitHubReview(crashed, target, a.client) + expect(a.calls.dismissReview).toHaveLength(0) + + const fileFailed = buildEnvelope({ + findings: [], + tier: "full", + mode: "gate", + scope: { unverified: [{ path: "models/m.sql", reason: "review failed for this file: boom" }] } as any, + }) + const b = fakeClient({ reviews: blocking }) + await postGitHubReview(fileFailed, target, b.client) + expect(b.calls.dismissReview).toHaveLength(0) + }) + test("a still-blocking rerun does not dismiss and posts REQUEST_CHANGES", async () => { const blocking = makeFinding({ severity: "critical", diff --git a/packages/opencode/test/altimate/review-scope-diff.test.ts b/packages/opencode/test/altimate/review-scope-diff.test.ts index 2c11c4bbc4..b4515e22be 100644 --- a/packages/opencode/test/altimate/review-scope-diff.test.ts +++ b/packages/opencode/test/altimate/review-scope-diff.test.ts @@ -337,3 +337,36 @@ describe("reviewPullRequest end to end: monorepo, PR into a non-default branch", expect(env.findings.some((f) => f.file.includes("drift"))).toBe(false) }, 60_000) }) + +describe("a shallow CI checkout without the target branch", () => { + test("the target branch and enough history are fetched to find the merge-base", async () => { + const exec = (await import("node:util")).promisify((await import("node:child_process")).execFile) + const fsp = (await import("node:fs")).promises + const os = await import("node:os") + const pth = await import("node:path") + const g = (cwd: string, ...a: string[]) => exec("git", a, { cwd }).then((r) => r.stdout.trim()) + const origin = await fsp.realpath(await fsp.mkdtemp(pth.join(os.tmpdir(), "shallow-origin-"))) + await g(origin, "init", "-q", "-b", "main") + await g(origin, "config", "user.email", "t@t") + await g(origin, "config", "user.name", "t") + await g(origin, "config", "commit.gpgsign", "false") + await g(origin, "config", "uploadpack.allowReachableSHA1InWant", "true") + for (let i = 0; i < 5; i++) { + await fsp.writeFile(pth.join(origin, `f${i}.sql`), `select ${i}\n`) + await g(origin, "add", "-A") + await g(origin, "commit", "-qm", `main ${i}`) + } + const branchPoint = await g(origin, "rev-parse", "HEAD") + await g(origin, "checkout", "-q", "-b", "feature") + for (let i = 0; i < 3; i++) { + await fsp.writeFile(pth.join(origin, `pr${i}.sql`), `select ${i}\n`) + await g(origin, "add", "-A") + await g(origin, "commit", "-qm", `pr ${i}`) + } + const clone = await fsp.realpath(await fsp.mkdtemp(pth.join(os.tmpdir(), "shallow-clone-"))) + await exec("git", ["clone", "-q", "--depth", "1", "--branch", "feature", `file://${origin}`, clone]) + const resolved = await resolveReviewBase({ cwd: clone, env: { GITHUB_BASE_REF: "main" } }) + expect(resolved.source).toBe("github_env") + expect(resolved.sha).toBe(branchPoint) + }, 60_000) +})