diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index cd1d9fc297..0952e234fd 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -199,6 +199,34 @@ jobs: # model_api_key: ${{ secrets.ANTHROPIC_API_KEY }} ``` +### What the review needs (and what each input unlocks) + +| Input | How to produce it | Unlocks | +|---|---|---| +| Full git history of the PR's target branch | `actions/checkout` with `fetch-depth: 0` | Correct base: the diff starts at the merge-base with the PR's target branch. A target branch that is not fetched is an error, never a silent fallback to `main`. | +| `target/manifest.json` | `dbt compile` (or `dbt parse` for lint + lineage only) | Blast radius, lineage breakage, tiering. Without it the run is lint-only. | +| `target/compiled/…` | `dbt compile` | Engine checks on the head (lint, grade, PII, grain). | +| `target-base/compiled/…` | Compile the base ref into `target-base/` (below) | Equivalence proofs and column-breakage for modified models. | +| `target/catalog.json` | `dbt docs generate` | Complete warehouse columns for column-lineage breakage. | + +Compiling the base (optional, for equivalence and column-breakage): + +```yaml + - name: Compile the base into target-base/ + run: | + git worktree add ../base "origin/${{ github.base_ref }}" + (cd ../base && dbt deps && dbt compile --target-path "$GITHUB_WORKSPACE/target-base") +``` + +**Monorepos:** set `project_dir` to the folder holding `dbt_project.yml`. Only +changes inside it are reviewed, `manifest_path` is resolved relative to it, and +changes elsewhere are counted in the comment. + +**What the comment tells you it could not check:** findings the engine could not +decide, models without compiled SQL, files outside the project, and whether the +advisory AI lane ran, was skipped, timed out or failed (with the reason). The +same facts are in the envelope (`scope`, `ai`) and the `review_run` telemetry. + ### Model & credentials for the advisory lane The deterministic engine (lineage, equivalence, PII, grade, lint — the only layer @@ -356,8 +384,9 @@ reviewer does **not** re-implement Jinja — it consumes dbt's own compiled outp from `target/compiled//…` (HEAD) and `target-base/compiled/…` (BASE), produced by `dbt compile`. To enable full equivalence verdicts, compile both the base and head refs in CI (the base into `target-base/`, the Recce - convention). Without compiled SQL the engine lanes fall back to raw and stay - *undecidable* (never fabricated) — the `dbt-patterns` lane still runs. + convention). A model without compiled SQL is never sent to the engine as raw + Jinja: it is listed as **not verified** (with the reason) in the comment and in + the envelope's `scope.unverified`, and the `dbt-patterns` lane still runs. ## What it checks — deterministic rule catalog diff --git a/github/review/action.yml b/github/review/action.yml index 8820ed8892..09c4bc1538 100644 --- a/github/review/action.yml +++ b/github/review/action.yml @@ -9,8 +9,12 @@ inputs: description: "comment (never blocks) | gate (blocks the merge on REQUEST_CHANGES)" required: false default: "comment" + project_dir: + description: "Directory of the dbt project (the one holding dbt_project.yml), relative to the repository root. Only changes inside it are reviewed — set it in a monorepo. `manifest_path` is resolved relative to it." + required: false + default: "." manifest_path: - description: "Path to the compiled dbt manifest.json. Run `dbt compile` before this step for the full (non-degraded) verdict." + description: "Path to the compiled dbt manifest.json, relative to `project_dir`. Run `dbt compile` before this step for the full (non-degraded) verdict; `dbt docs generate` (catalog.json) and a base compile into `target-base/` enable column-lineage breakage and equivalence proofs." required: false default: "target/manifest.json" severity_threshold: @@ -170,6 +174,7 @@ runs: # from `${{ }}` would be a shell-injection vector. Build an arg array instead. env: IN_MODE: ${{ inputs.mode }} + IN_PROJECT_DIR: ${{ inputs.project_dir }} IN_MANIFEST: ${{ inputs.manifest_path }} IN_SEVERITY: ${{ inputs.severity_threshold }} IN_BASE: ${{ inputs.base }} @@ -181,6 +186,7 @@ runs: ALTIMATE_REVIEW_SIGNING_KEY: ${{ inputs.signing_key }} run: | args=(--mode "$IN_MODE" --manifest "$IN_MANIFEST" --severity "$IN_SEVERITY") + [[ -n "$IN_PROJECT_DIR" && "$IN_PROJECT_DIR" != "." ]] && args+=(--cwd "$IN_PROJECT_DIR") [[ -n "$IN_BASE" ]] && args+=(--base "$IN_BASE") [[ -n "$IN_HEAD" ]] && args+=(--head "$IN_HEAD") [[ "$IN_POST" == "true" ]] && args+=(--post) diff --git a/packages/opencode/src/altimate/review/ai-review.ts b/packages/opencode/src/altimate/review/ai-review.ts index 2ea19b0bd7..bd59dd54ed 100644 --- a/packages/opencode/src/altimate/review/ai-review.ts +++ b/packages/opencode/src/altimate/review/ai-review.ts @@ -11,7 +11,8 @@ import { readStreamText } from "../llm/stream-text" const log = Log.create({ service: "ai-review" }) -const AI_TIMEOUT_MS = 60_000 +const AI_TIMEOUT_BASE_MS = 60_000 +const AI_TIMEOUT_MAX_MS = 300_000 const MAX_DIFF_CHARS = 6_000 // per file, keep the prompt bounded const MAX_FILES = 20 @@ -31,6 +32,39 @@ export interface AiReviewInput { grounding: Finding[] prTitle?: string prBody?: string + /** Explicit `provider/model`; omit to use the configured default model. */ + model?: string +} + +/** What happened in the advisory lane, reported on the envelope and in the comment. */ +export type AiLaneState = "ok" | "skipped" | "timeout" | "error" +export interface AiLaneStatus { + state: AiLaneState + /** Why the lane was skipped or failed (never contains credentials). */ + reason?: string + /** `provider/model` the lane ran with. */ + model?: string + durationMs?: number +} +export interface AiReviewResult { + findings: Finding[] + status: AiLaneStatus +} + +/** Timeout scaled to the prompt: 60 s plus 1 s per 2,000 characters, capped at 5 minutes. */ +export function aiTimeoutMs(promptChars: number): number { + return Math.min(AI_TIMEOUT_MAX_MS, AI_TIMEOUT_BASE_MS + Math.ceil(promptChars / 2_000) * 1_000) +} + +/** Error text safe to show in a PR comment: one line, bounded, with anything + * that looks like a key or bearer token masked. */ +function safeReason(err: unknown): string { + const raw = err instanceof Error ? err.message : String(err) + return raw + .replace(/\s+/g, " ") + .replace(/(bearer\s+)[^\s]+/gi, "$1***") + .replace(/\b(sk|pk|key|token)[-_][A-Za-z0-9_-]{8,}/gi, "***") + .slice(0, 200) } /** @@ -73,23 +107,39 @@ function buildUserMessage(input: AiReviewInput): string { /** * Run the LLM reviewer lane. Returns advisory findings (severity ≤ warning, - * clamped by core), or [] if no model / core is available or the call fails — - * a review must never crash because the AI layer is unavailable. + * clamped by core) and a status saying whether the lane ran, was skipped, timed + * out or failed. Never throws — a review must not crash because the AI layer is + * unavailable — but never fails silently either: the status reaches the + * envelope and the PR comment. */ -export async function runAiReview(input: AiReviewInput): Promise { +export async function runAiReview(input: AiReviewInput): Promise { + const startedAt = Date.now() + const done = (findings: Finding[], status: Omit): AiReviewResult => ({ + findings, + status: { ...status, durationMs: Date.now() - startedAt }, + }) const files = input.files.filter((f) => f.status !== "deleted" && (f.diff || f.sql)) - if (!files.length) return [] + if (!files.length) return done([], { state: "skipped", reason: "no changed model SQL to review" }) + const message = buildUserMessage({ ...input, files }) const controller = new AbortController() - const timeout = setTimeout(() => controller.abort(), AI_TIMEOUT_MS) + const timeoutMs = aiTimeoutMs(message.length) + const timeout = setTimeout(() => controller.abort(), timeoutMs) + let modelLabel: string | undefined try { // Prompt comes from the compiled core, not this file. const promptRes = await Dispatcher.call("altimate_core.review_ai_prompt", {}) const system = ((promptRes.data ?? {}) as Record).prompt as string | undefined - if (!system) return [] - - const defaultModel = await Provider.defaultModel() - const model = await Provider.getModel(defaultModel.providerID, defaultModel.modelID) + if (!system) return done([], { state: "error", reason: "the core reviewer prompt is unavailable" }) + + let model: Awaited> + try { + const chosen = input.model ? Provider.parseModel(input.model) : await Provider.defaultModel() + model = await Provider.getModel(chosen.providerID, chosen.modelID) + } catch (err) { + return done([], { state: "skipped", reason: `no usable model: ${safeReason(err)}`, model: input.model }) + } + modelLabel = `${model.providerID}/${model.id}` const agent: Agent.Info = { name: "dbt-ai-reviewer", @@ -119,10 +169,12 @@ export async function runAiReview(input: AiReviewInput): Promise { abort: controller.signal, sessionID: user.sessionID, retries: 1, - messages: [{ role: "user", content: buildUserMessage({ ...input, files }) }], + messages: [{ role: "user", content: message }], }) const text = await readStreamText(stream) - if (!text) return [] + if (controller.signal.aborted) + return done([], { state: "timeout", reason: `no answer within ${Math.round(timeoutMs / 1000)} s`, model: modelLabel }) + if (!text) return done([], { state: "error", reason: "the model returned no text", model: modelLabel }) // Parse + clamp in core (the prompt-injection-resistant, advisory-only // contract). Returns already-validated, severity-clamped, file-checked items. @@ -165,10 +217,12 @@ export async function runAiReview(input: AiReviewInput): Promise { } } log.info("ai review complete", { findings: out.length }) - return out + return done(out, { state: "ok", model: modelLabel }) } catch (err) { log.error("ai review failed", { error: err }) - return [] + if (controller.signal.aborted) + return done([], { state: "timeout", reason: `no answer within ${Math.round(timeoutMs / 1000)} s`, model: modelLabel }) + return done([], { state: "error", reason: safeReason(err), model: modelLabel }) } finally { clearTimeout(timeout) } diff --git a/packages/opencode/src/altimate/review/format.ts b/packages/opencode/src/altimate/review/format.ts index 0befe5883c..a110068e8b 100644 --- a/packages/opencode/src/altimate/review/format.ts +++ b/packages/opencode/src/altimate/review/format.ts @@ -42,11 +42,12 @@ export function renderSummary(env: VerdictEnvelope): string { if (env.summary.degraded) { lines.push( - "> ⚙️ **Lint-only run** — no dbt manifest/warehouse was available, so lineage, equivalence and", + "> ⚙️ **Lint-only run** — no changed model resolved against a dbt manifest, so lineage, equivalence and", "> data-impact checks were skipped. Wire `manifest_path` (and optionally warehouse creds) for the full verdict.", "", ) } + lines.push(...coverageNotes(env)) // G1 (Round 18) — surface classifier reasons when --explain-tier populated // them, so a customer can see why the review ran at this tier. Also surfaces @@ -110,11 +111,41 @@ export function renderSummary(env: VerdictEnvelope): string { `altimate dbt-pr-review · verdict \`${env.verdict}\`` + (env.signature ? ` · signed \`${env.signature.slice(0, 18)}…\`` : "") + (env.manifestHash ? ` · manifest \`${env.manifestHash.slice(0, 10)}\`` : "") + + (env.scope?.baseRef ? ` · base \`${env.scope.baseRef}\`` : "") + + (env.ai?.state === "ok" && env.ai.model ? ` · AI \`${env.ai.model}\`` : "") + "", ) return lines.join("\n") } +/** + * What the review could not fully check, each on its own line: undecidable + * findings, models without compiled SQL, files outside the dbt project, and + * the advisory AI lane when it did not run. Kept separate from the lint-only + * banner so one undecidable finding never mislabels a manifest-backed run. + */ +export function coverageNotes(env: VerdictEnvelope): string[] { + const notes: string[] = [] + const undecided = env.findings.filter((f) => f.degraded).length + if (undecided) notes.push(`${undecided} finding(s) could not be decided by the engine and are marked _unverified_.`) + const unverified = env.scope?.unverified ?? [] + if (unverified.length) { + const head = unverified.slice(0, 5).map((u) => `\`${u.path}\``).join(", ") + const more = unverified.length > 5 ? ` and ${unverified.length - 5} more` : "" + notes.push( + `${unverified.length} model(s) not verified by the engine — ${unverified[0].reason}: ${head}${more}. ` + + "Run `dbt compile` (and compile the base into `target-base/`) for equivalence and lineage checks.", + ) + } + if (env.scope?.outOfScopeFiles) + notes.push(`${env.scope.outOfScopeFiles} changed file(s) outside the dbt project were not reviewed.`) + if (env.ai && env.ai.state !== "ok") { + const what = { skipped: "skipped", timeout: "timed out", error: "failed" }[env.ai.state] + notes.push(`Advisory AI review ${what}${env.ai.reason ? `: ${env.ai.reason}` : ""}.`) + } + return notes.length ? [...notes.map((n) => `> ℹ️ ${n}`), ""] : [] +} + /** GitHub `pulls.createReview` inline comments — only findings with a line. */ export interface InlineComment { path: string diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 2c81fcc721..48e979e1ff 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -14,7 +14,7 @@ import { type Rubric, exclusionReason, clampSeverity } from "./rubric" import { type ReviewConfig } from "./config" import { type ReviewMode, type ReviewScope, type VerdictEnvelope, buildEnvelope, signEnvelope } from "./verdict" import { detectModelPatterns, detectSchemaYmlPatterns, splitDiff } from "./dbt-patterns" -import { type AiReviewInput } from "./ai-review" +import { type AiLaneStatus, type AiReviewInput, type AiReviewResult } from "./ai-review" /** * The deterministic review recipe. @@ -188,7 +188,11 @@ export interface OrchestrateInput { * tests pass a fake or omit it (the lane is then skipped). Receives the * deterministic findings as grounding and returns ADVISORY findings only. */ - aiReview?: (input: AiReviewInput) => Promise + aiReview?: (input: AiReviewInput) => Promise + /** Why the AI lane is off when `aiReview` is not injected (shown in the comment). */ + aiDisabledReason?: string + /** Explicit `provider/model` for the AI lane. */ + aiModel?: string /** PR metadata passed to the AI reviewer for intent checking. */ prTitle?: string prBody?: string @@ -1410,7 +1414,12 @@ export async function runReview(input: OrchestrateInput): Promise ({ path: ctx.file.path, status: ctx.file.status, @@ -1419,16 +1428,20 @@ export async function runReview(input: OrchestrateInput): Promise SEVERITY_ORDER[b.severity] - SEVERITY_ORDER[a.severity] || a.file.localeCompare(b.file)) - const degraded = runDegraded || findings.some((f) => f.degraded) + // Lint-only means no changed model resolved against a manifest. A single + // undecidable finding does not make the whole run lint-only; those findings + // carry their own `degraded` flag and are counted separately in the summary. + const degraded = runDegraded const envelope = buildEnvelope({ + ai: aiStatus, findings, tier, mode: input.mode, diff --git a/packages/opencode/src/altimate/review/run.ts b/packages/opencode/src/altimate/review/run.ts index b315c550dc..5bd581f204 100644 --- a/packages/opencode/src/altimate/review/run.ts +++ b/packages/opencode/src/altimate/review/run.ts @@ -53,6 +53,8 @@ export interface ReviewPullRequestOptions { /** PR metadata for the AI reviewer's intent check. */ prTitle?: string prBody?: string + /** Explicit `provider/model` for the advisory AI lane. */ + aiModel?: string /** G1 — surface the tier classifier's reasons on the verdict envelope. */ explainTier?: boolean /** G2 — [EXPERIMENTAL] bypass the tier classifier with the supplied tier. @@ -379,6 +381,8 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise // path verdicts (cubic-review PR #1041). cliVersion: opts.cliVersion ?? Installation.VERSION, aiReview: opts.noAi || config.ai === false ? undefined : runAiReview, + aiDisabledReason: opts.noAi ? "disabled by --no-ai" : config.ai === false ? "disabled by `ai: false` in .altimate/review.yml" : undefined, + aiModel: opts.aiModel, prTitle: opts.prTitle, prBody: opts.prBody, explainTier: opts.explainTier, @@ -396,6 +400,32 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise }) } +/** + * The pull/merge request's title and description from the CI environment, so + * the AI lane can check the change against its stated intent. GitHub: the + * `pull_request` object in GITHUB_EVENT_PATH. GitLab: CI_MERGE_REQUEST_TITLE / + * CI_MERGE_REQUEST_DESCRIPTION. Empty outside a PR/MR run. + */ +export async function prContextFromEnv(env: NodeJS.ProcessEnv = process.env): Promise<{ prTitle?: string; prBody?: string }> { + if (env.GITHUB_EVENT_PATH) { + try { + const pr = JSON.parse(await readFile(env.GITHUB_EVENT_PATH, "utf8"))?.pull_request + if (pr) { + return { + prTitle: typeof pr.title === "string" && pr.title ? pr.title : undefined, + prBody: typeof pr.body === "string" && pr.body ? pr.body : undefined, + } + } + } catch { + /* unreadable event file: try GitLab */ + } + } + return { + prTitle: env.CI_MERGE_REQUEST_TITLE || undefined, + prBody: env.CI_MERGE_REQUEST_DESCRIPTION || undefined, + } +} + /** * The dbt project being reviewed: the nearest directory at or above `cwd` * (not above the repo root) holding dbt_project.yml; else the project that diff --git a/packages/opencode/src/altimate/review/telemetry.ts b/packages/opencode/src/altimate/review/telemetry.ts index fc20cc3b17..34016e8a50 100644 --- a/packages/opencode/src/altimate/review/telemetry.ts +++ b/packages/opencode/src/altimate/review/telemetry.ts @@ -110,6 +110,12 @@ export function emitReviewRun(input: { // Optional in the schema and explicitly invalid as `false`, so normalise rather than copy. tier_forced: env.tierForced === true, degraded: env.summary.degraded, + undecided: env.findings.filter((f) => f.degraded).length, + unverified_models: env.scope?.unverified?.length ?? 0, + ai_state: env.ai?.state, + ai_duration_ms: env.ai?.durationMs, + base_source: env.scope?.baseSource, + out_of_scope_files: env.scope?.outOfScopeFiles ?? 0, stale_manifest: env.staleManifest === true, critical: env.summary.critical, warning: env.summary.warning, diff --git a/packages/opencode/src/altimate/review/verdict.ts b/packages/opencode/src/altimate/review/verdict.ts index 4a2fa372a7..655404de6a 100644 --- a/packages/opencode/src/altimate/review/verdict.ts +++ b/packages/opencode/src/altimate/review/verdict.ts @@ -144,6 +144,15 @@ export const VerdictEnvelope = z.object({ staleManifest: z.boolean().optional(), /** Base resolution and review coverage (see ReviewScope). */ scope: ReviewScope.optional(), + /** Advisory AI lane outcome: ran, skipped, timed out or failed, and why. */ + ai: z + .object({ + state: z.enum(["ok", "skipped", "timeout", "error"]), + reason: z.string().optional(), + model: z.string().optional(), + durationMs: z.number().int().nonnegative().optional(), + }) + .optional(), /** ISO timestamp; injected by the caller (no clock access in pure code). */ generatedAt: z.string().optional(), /** Optional break-glass override record. */ @@ -203,6 +212,7 @@ export interface BuildEnvelopeInput { /** True when mtime signals the manifest predates a change-affecting file. */ staleManifest?: boolean scope?: ReviewScope + ai?: VerdictEnvelope["ai"] } function summarize(findings: Finding[], degraded: boolean): VerdictEnvelope["summary"] { @@ -232,6 +242,7 @@ export function buildEnvelope(input: BuildEnvelopeInput): VerdictEnvelope { manifestHash: input.manifestHash, staleManifest: input.staleManifest ? true : undefined, scope: input.scope, + ai: input.ai, generatedAt: input.generatedAt, }) } diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 750f641264..8566507850 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -1594,10 +1594,20 @@ export namespace Telemetry { mode?: string tier?: string tier_forced?: boolean - /** The envelope's fidelity flag: no reviewable files, no usable manifest for the changed - * models, OR a surfaced finding whose engine analysis was undecidable. It does NOT mean - * merely "no warehouse". */ + /** The envelope's fidelity flag: no reviewable files, or no changed model resolved against + * a manifest (lint-only). An undecidable finding alone no longer sets it — see + * `undecided`. It does NOT mean merely "no warehouse". */ degraded?: boolean + /** Surfaced findings whose engine analysis was undecidable. */ + undecided?: number + /** Models the engine could not verify (no compiled SQL). */ + unverified_models?: number + /** Advisory AI lane outcome: ok | skipped | timeout | error. */ + ai_state?: string + ai_duration_ms?: number + /** Where the review base came from (flag, github_event, default_branch, …). */ + base_source?: string + out_of_scope_files?: number stale_manifest?: boolean critical?: number warning?: number diff --git a/packages/opencode/src/cli/cmd/review.ts b/packages/opencode/src/cli/cmd/review.ts index 7897396ac5..e35556aa0b 100644 --- a/packages/opencode/src/cli/cmd/review.ts +++ b/packages/opencode/src/cli/cmd/review.ts @@ -3,7 +3,7 @@ import { UI } from "../ui" import { cmd } from "./cmd" import { bootstrap } from "../bootstrap" import { Installation } from "../../installation" -import { reviewPullRequest } from "../../altimate/review/run" +import { prContextFromEnv, reviewPullRequest } from "../../altimate/review/run" import { renderSummary } from "../../altimate/review/format" import { postGitHubReview, resolveGitHubTarget } from "../../altimate/review/post-github" // altimate_change — review feature telemetry @@ -66,6 +66,10 @@ export const ReviewCommand = cmd({ choices: ["trivial", "lite", "full"] as const, describe: "[EXPERIMENTAL / bench debug] override the tier classifier — verdict envelope carries tierForced: true", }) + .option("ai-model", { + type: "string", + describe: "provider/model for the advisory AI lane (default: the configured default model)", + }) .option("cwd", { type: "string", describe: "project directory (default: current dir)" }), async handler(args) { const cwd = (args.cwd as string) || process.cwd() @@ -92,9 +96,13 @@ export const ReviewCommand = cmd({ // altimate_change — time the engine only. Output writing and posting happen after this and // must not be counted as review latency, nor turn a computed review into a failed one. const startedAt = Date.now() + // PR title/description from the CI event, for the AI lane's intent check. + const prContext = await prContextFromEnv() let env try { env = await reviewPullRequest({ + ...prContext, + aiModel: args.aiModel as string | undefined, cwd, base: args.base as string | undefined, head: args.head as string | undefined, diff --git a/packages/opencode/test/altimate/review-ci-honesty.test.ts b/packages/opencode/test/altimate/review-ci-honesty.test.ts new file mode 100644 index 0000000000..7042d76e24 --- /dev/null +++ b/packages/opencode/test/altimate/review-ci-honesty.test.ts @@ -0,0 +1,188 @@ +import { describe, test, expect } from "bun:test" +import { promises as fs } from "node:fs" +import path from "node:path" +import { tmpdir } from "node:os" +import { runReview, type ReviewRunner } from "../../src/altimate/review/orchestrate" +import { DEFAULT_REVIEW_CONFIG } from "../../src/altimate/review/config" +import { DEFAULT_RUBRIC } from "../../src/altimate/review/rubric" +import { makeFinding } from "../../src/altimate/review/finding" +import { aiTimeoutMs } from "../../src/altimate/review/ai-review" +import { coverageNotes, renderSummary } from "../../src/altimate/review/format" +import { prContextFromEnv } from "../../src/altimate/review/run" +import { verifyEnvelope } from "../../src/altimate/review/verdict" + +/** + * The PR comment must be honest about what was checked: a manifest-backed run + * is not "lint-only" because one finding was undecidable, the advisory AI lane + * never fails silently, and the PR's title/description reach the intent check. + */ + +const runner: ReviewRunner = { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return { grade: "B" } + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + // Undecidable → the semantic lane emits a `degraded` finding. + return { decided: false } + }, + async detectPii() { + return { columns: [] } + }, +} as ReviewRunner + +const modified = [{ path: "models/marts/m.sql", status: "modified" as const, diff: "-select 1 as id\n+select 2 as id\n" }] +const content = async (_f: string, side: "old" | "new") => (side === "old" ? "select 1 as id" : "select 2 as id") + +function review(extra: Partial[0]> = {}) { + return runReview({ + changedFiles: modified, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["semantic_change", "ai_review"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner, + getContent: content, + getCompiled: content, + generatedAt: "2026-11-12T00:00:00Z", + ...extra, + }) +} + +describe("lint-only banner", () => { + test("an undecidable finding on a manifest-backed run does not flip the run to lint-only", async () => { + const env = await review() + expect(env.findings.some((f) => f.degraded)).toBe(true) + expect(env.summary.degraded).toBe(false) + const md = renderSummary(env) + expect(md).not.toContain("Lint-only run") + expect(md).toContain("could not be decided by the engine") + }) + + test("no manifest for the changed models is still lint-only", async () => { + const env = await review({ + runner: { + ...runner, + async impact() { + return { hasManifest: false, severity: "UNKNOWN", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + } as ReviewRunner, + }) + expect(env.summary.degraded).toBe(true) + expect(renderSummary(env)).toContain("Lint-only run") + }) +}) + +describe("advisory AI lane status", () => { + test("disabled lane is reported as skipped with the reason", async () => { + const env = await review({ aiDisabledReason: "disabled by --no-ai" }) + expect(env.ai).toEqual({ state: "skipped", reason: "disabled by --no-ai" }) + expect(coverageNotes(env).join("\n")).toContain("Advisory AI review skipped: disabled by --no-ai") + }) + + test("lane outside the tier is reported as skipped", async () => { + const env = await review({ config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["semantic_change"] } }) + expect(env.ai?.state).toBe("skipped") + expect(env.ai?.reason).toContain("tier") + }) + + test("a timeout is surfaced, signed, and never blocks", async () => { + const env = await review({ + aiReview: async () => ({ findings: [], status: { state: "timeout", reason: "no answer within 75 s", durationMs: 75000 } }), + }) + expect(env.ai?.state).toBe("timeout") + expect(verifyEnvelope(env)).toBe(true) + expect(renderSummary(env)).toContain("Advisory AI review timed out: no answer within 75 s") + }) + + test("a throwing lane becomes an error status instead of vanishing", async () => { + const env = await review({ + aiReview: async () => { + throw new Error("provider exploded") + }, + }) + expect(env.ai).toEqual({ state: "error", reason: "provider exploded" }) + }) + + test("a successful lane is ok, its model goes in the footer, and the model selector is passed through", async () => { + let sawModel: string | undefined + const env = await review({ + aiModel: "anthropic/claude-sonnet-5-5", + aiReview: async (input) => { + sawModel = input.model + return { + findings: [ + makeFinding({ + severity: "critical", + category: "semantic_change", + title: "m: intent mismatch", + body: "x", + file: "models/marts/m.sql", + model: "m", + confidence: "medium", + evidence: { tool: "ai-review", result: {} }, + ruleKey: "ai:t", + }), + ], + status: { state: "ok", model: "anthropic/claude-sonnet-5-5", durationMs: 1200 }, + } + }, + }) + expect(sawModel).toBe("anthropic/claude-sonnet-5-5") + expect(env.ai?.state).toBe("ok") + // Clamped: the AI lane never blocks. + expect(env.findings.find((f) => f.evidence?.tool === "ai-review")?.severity).toBe("warning") + expect(renderSummary(env)).toContain("AI `anthropic/claude-sonnet-5-5`") + expect(coverageNotes(env).join("\n")).not.toContain("Advisory AI review") + }) + + test("legacy lanes returning a bare findings array still work", async () => { + const env = await review({ aiReview: async () => [] }) + expect(env.ai?.state).toBe("ok") + }) + + test("timeout scales with the prompt and is capped", () => { + expect(aiTimeoutMs(0)).toBe(60_000) + expect(aiTimeoutMs(30_000)).toBe(75_000) + expect(aiTimeoutMs(10_000_000)).toBe(300_000) + }) +}) + +describe("PR title and description reach the intent check", () => { + test("GitHub: read from the pull_request event", async () => { + const dir = await fs.mkdtemp(path.join(tmpdir(), "review-event-")) + const ev = path.join(dir, "event.json") + await fs.writeFile(ev, JSON.stringify({ pull_request: { title: "Add fct_orders", body: "## Spec\n- grain: order_id" } })) + expect(await prContextFromEnv({ GITHUB_EVENT_PATH: ev })).toEqual({ + prTitle: "Add fct_orders", + prBody: "## Spec\n- grain: order_id", + }) + }) + + test("GitLab: read from the merge-request variables", async () => { + expect( + await prContextFromEnv({ CI_MERGE_REQUEST_TITLE: "Fix grain", CI_MERGE_REQUEST_DESCRIPTION: "dedupe on id" }), + ).toEqual({ prTitle: "Fix grain", prBody: "dedupe on id" }) + }) + + test("outside a PR run there is no context", async () => { + expect(await prContextFromEnv({})).toEqual({ prTitle: undefined, prBody: undefined }) + }) + + test("the context is forwarded to the AI lane", async () => { + let seen: { prTitle?: string; prBody?: string } = {} + await review({ + prTitle: "Add fct_orders", + prBody: "grain: order_id", + aiReview: async (input) => { + seen = { prTitle: input.prTitle, prBody: input.prBody } + return [] + }, + }) + expect(seen).toEqual({ prTitle: "Add fct_orders", prBody: "grain: order_id" }) + }) +})