From f08152c16eacb1fe74139bc9c37d8def63d4b048 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 13 Nov 2026 13:23:09 +0530 Subject: [PATCH 1/3] feat(review): optional `dbt_prepare` in the action builds the head, merge-base and catalog artifacts - new action inputs: `dbt_prepare` (`none` default | `compile` | `full`), `dbt_command`, `dbt_profiles_dir` - `compile` runs `dbt compile` on the head and compiles the PR's merge-base into `target-base/` in a temporary worktree (fetching and deepening a shallow checkout when needed); `full` adds `dbt docs generate` - a base compile that fails is a warning: the review still runs and marks those models unverified - inputs reach the script only through env, never interpolated - docs: the manual base-compile snippet now compiles the merge-base, not the base branch's tip - the example workflow is renamed `examples/dbt-pr-review.yml` (was named after one internal repo) Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 24 +++++++- github/review/action.yml | 59 +++++++++++++++++++ ...timate-ingestion.yml => dbt-pr-review.yml} | 2 +- .../test/altimate/review-action.test.ts | 24 ++++++++ .../skill/release-v0.8.5-adversarial.test.ts | 2 +- 5 files changed, 106 insertions(+), 5 deletions(-) rename github/review/examples/{altimate-ingestion.yml => dbt-pr-review.yml} (97%) create mode 100644 packages/opencode/test/altimate/review-action.test.ts diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index 332d26d0b3..4c344ee9ca 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -209,12 +209,30 @@ jobs: | `target-base/compiled/…` | Compile the base ref into `target-base/` (below) | Equivalence proofs and column-breakage for modified models. | | `target/catalog.json` | `dbt docs generate` | Complete warehouse columns for column-lineage breakage. | -Compiling the base (optional, for equivalence and column-breakage): +The action can build these for you with `dbt_prepare` (off by default): + +```yaml + - uses: AltimateAI/altimate-code/github/review@main + with: + dbt_prepare: compile # none | compile | full + dbt_command: dbt # or "uv run dbt", "poetry run dbt" + dbt_profiles_dir: . # relative to project_dir +``` + +- **`compile`:** runs `dbt compile` on the head, then compiles the PR's merge-base + into `target-base/` in a temporary worktree. This enables equivalence and + column-breakage proofs. It usually works offline. +- **`full`:** also runs `dbt docs generate` for `catalog.json` (complete column + types). That needs warehouse credentials in the profile. +- A base compile that fails is a warning: the review still runs, and the + affected models are listed as not verified. + +To build them yourself, compile the **merge-base**, not the base branch's tip: ```yaml - name: Compile the base into target-base/ run: | - git worktree add ../base "origin/${{ github.base_ref }}" + git worktree add ../base "$(git merge-base origin/${{ github.base_ref }} HEAD)" (cd ../base && dbt deps && dbt compile --target-path "$GITHUB_WORKSPACE/target-base") ``` @@ -242,7 +260,7 @@ ways, or neither (it self-disables, leaving a deterministic-only review): Always pass keys as repo **secrets**. Warehouse credentials are consumed by the `dbt compile` step (via your `profiles.yml`), **not** by the review step. A complete, copy-paste workflow lives at -[`github/review/examples/altimate-ingestion.yml`](https://github.com/AltimateAI/altimate-code/blob/main/github/review/examples/altimate-ingestion.yml). +[`github/review/examples/dbt-pr-review.yml`](https://github.com/AltimateAI/altimate-code/blob/main/github/review/examples/dbt-pr-review.yml). Re-pushing commits updates the same summary comment in place; fixed findings are dropped on the next run. `--post` targets **GitHub** PRs (it reads `GITHUB_TOKEN` / diff --git a/github/review/action.yml b/github/review/action.yml index 09c4bc1538..f39818fc6b 100644 --- a/github/review/action.yml +++ b/github/review/action.yml @@ -31,6 +31,17 @@ inputs: description: "Post the verdict to the PR (summary comment + inline review)." required: false default: "true" + dbt_prepare: + description: "Build the dbt artifacts the full review uses before reviewing. `none` (default): you build them in earlier steps. `compile`: `dbt compile` on the head, and on the PR's merge-base into `target-base/` (equivalence and lineage proofs). `full`: also `dbt docs generate` for `catalog.json` (complete column types). `compile` usually works offline; `full` needs warehouse credentials in the profile." + required: false + default: "none" + dbt_command: + description: "How to invoke dbt for `dbt_prepare`, e.g. `dbt`, `uv run dbt`, `poetry run dbt`." + required: false + default: "dbt" + dbt_profiles_dir: + description: "`--profiles-dir` for `dbt_prepare`, relative to `project_dir`. Empty = dbt's default lookup." + required: false signing_key: description: "Optional HMAC key to sign the verdict envelope (ALTIMATE_REVIEW_SIGNING_KEY)." required: false @@ -167,6 +178,54 @@ runs: echo "Advisory lane: no model/credentials provided — running deterministic-only." fi + # Optional: build the head and base artifacts the full-tier review needs. + # Inputs are passed as ENV, never interpolated into the script. + - name: Prepare dbt artifacts + if: inputs.dbt_prepare != 'none' + shell: bash + env: + IN_PREPARE: ${{ inputs.dbt_prepare }} + IN_DBT: ${{ inputs.dbt_command }} + IN_PROFILES: ${{ inputs.dbt_profiles_dir }} + IN_PROJECT_DIR: ${{ inputs.project_dir }} + IN_BASE: ${{ inputs.base }} + EVENT_BASE: ${{ github.event.pull_request.base.ref }} + run: | + set -euo pipefail + case "$IN_PREPARE" in compile|full) ;; *) echo "::error::dbt_prepare must be none, compile or full (got '$IN_PREPARE')"; exit 1 ;; esac + read -ra DBT <<< "$IN_DBT" + PROJECT="$(cd "${IN_PROJECT_DIR:-.}" && pwd)" + REL="$(git -C "$PROJECT" rev-parse --show-prefix)" + PROFILES=() + [[ -n "${IN_PROFILES:-}" ]] && PROFILES=(--profiles-dir "$(cd "$PROJECT" && cd "$IN_PROFILES" && pwd)") + build() { # $1 = project dir, $2 = target path + ( cd "$1" + [[ -f packages.yml || -f dependencies.yml ]] && "${DBT[@]}" deps ${PROFILES[@]+"${PROFILES[@]}"} + "${DBT[@]}" compile --target-path "$2" ${PROFILES[@]+"${PROFILES[@]}"} ) + } + echo "Compiling the head…" + build "$PROJECT" "$PROJECT/target" + if [[ "$IN_PREPARE" == "full" ]]; then + echo "Generating catalog.json…" + ( cd "$PROJECT" && "${DBT[@]}" docs generate --target-path "$PROJECT/target" ${PROFILES[@]+"${PROFILES[@]}"} ) + fi + # The base side: the PR's merge-base, compiled into target-base/. + BASE="${IN_BASE:-$EVENT_BASE}" + BASE="${BASE#origin/}" + if [[ -z "$BASE" ]]; then echo "::warning::No base branch known; skipping the base compile."; exit 0; fi + git fetch --no-tags origin "+refs/heads/$BASE:refs/remotes/origin/$BASE" >/dev/null 2>&1 || true + MB="$(git merge-base "origin/$BASE" HEAD 2>/dev/null || git merge-base "$BASE" HEAD 2>/dev/null || true)" + if [[ -z "$MB" && "$(git rev-parse --is-shallow-repository)" == "true" ]]; then + git fetch --no-tags --deepen=1000 origin "+refs/heads/$BASE:refs/remotes/origin/$BASE" "$(git rev-parse HEAD)" >/dev/null 2>&1 || true + MB="$(git merge-base "origin/$BASE" HEAD 2>/dev/null || true)" + fi + if [[ -z "$MB" ]]; then echo "::warning::Could not find the merge-base with $BASE; skipping the base compile."; exit 0; fi + WT="$RUNNER_TEMP/altimate-review-base" + git worktree add --detach "$WT" "$MB" >/dev/null + trap 'git worktree remove --force "$WT" >/dev/null 2>&1 || true' EXIT + echo "Compiling the base ($BASE @ ${MB:0:12})…" + build "$WT/$REL" "$PROJECT/target-base" || echo "::warning::The base compile failed; equivalence will be marked unverified." + - name: Run dbt PR review shell: bash # Inputs are passed as ENV (data), never interpolated into the command — diff --git a/github/review/examples/altimate-ingestion.yml b/github/review/examples/dbt-pr-review.yml similarity index 97% rename from github/review/examples/altimate-ingestion.yml rename to github/review/examples/dbt-pr-review.yml index dc52867909..799d424424 100644 --- a/github/review/examples/altimate-ingestion.yml +++ b/github/review/examples/dbt-pr-review.yml @@ -1,4 +1,4 @@ -# Example: drop-in dbt PR review for a dbt repo (e.g. altimate-ingestion). +# Example: drop-in dbt PR review for a dbt repo. # Copy to the target repo as `.github/workflows/dbt-pr-review.yml` and adjust the # dbt adapter + profiles/secrets to match the warehouse. # diff --git a/packages/opencode/test/altimate/review-action.test.ts b/packages/opencode/test/altimate/review-action.test.ts new file mode 100644 index 0000000000..36f0417224 --- /dev/null +++ b/packages/opencode/test/altimate/review-action.test.ts @@ -0,0 +1,24 @@ +import { describe, test, expect } from "bun:test" +import { promises as fs } from "node:fs" +import path from "node:path" +import YAML from "yaml" + +/** The GitHub Action's optional artifact preparation stays opt-in and injection-safe. */ +const actionPath = path.join(import.meta.dir, "../../../../github/review/action.yml") + +describe("action: dbt_prepare", () => { + test("is opt-in, and its script reads inputs only from env", async () => { + const action = YAML.parse(await fs.readFile(actionPath, "utf8")) + expect(action.inputs.dbt_prepare.default).toBe("none") + expect(action.inputs.dbt_command.default).toBe("dbt") + const steps: Array<{ name: string; if?: string; run?: string; env?: Record }> = action.runs.steps + const prep = steps.find((s) => s.name === "Prepare dbt artifacts")! + const review = steps.findIndex((s) => s.name === "Run dbt PR review") + expect(steps.indexOf(prep)).toBeLessThan(review) + expect(prep.if).toBe("inputs.dbt_prepare != 'none'") + expect(prep.run).not.toContain("${{") + expect(prep.env?.IN_DBT).toBe("${{ inputs.dbt_command }}") + // The base is compiled into target-base/, where the compiled-SQL resolver looks. + expect(prep.run).toContain('"$PROJECT/target-base"') + }) +}) diff --git a/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts b/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts index 00b1ee1724..77d91cbf04 100644 --- a/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts +++ b/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts @@ -238,7 +238,7 @@ describe("v0.8.5 adversarial - composite action", () => { const changelog = await fs.readFile(path.join(repoRoot, "CHANGELOG.md"), "utf8") expect(changelog).toContain(`## [${version}]`) - for (const relative of ["docs/docs/usage/dbt-pr-review.md", "github/review/examples/altimate-ingestion.yml"]) { + for (const relative of ["docs/docs/usage/dbt-pr-review.md", "github/review/examples/dbt-pr-review.yml"]) { const content = await fs.readFile(path.join(repoRoot, relative), "utf8") expect(content).toContain(`AltimateAI/altimate-code/github/review@v${version}`) expect(content).not.toContain("AltimateAI/altimate-code/github/review@v0.8.4") From 2e7176ffb3a44b2ccb4d783a696f81ea4db67694 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 13 Nov 2026 13:29:00 +0530 Subject: [PATCH 2/3] fix(review): column-set rules work in `dbt parse`-only CI; dedup windows split across lines are read whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by replaying the PRs behind known escaped defects: - without compiled SQL, output columns come from the raw model's final SELECT (Jinja blanked; a Jinja expression in the list makes it unknown). `on_schema_change='fail'` with new columns now fires on the two real escapes it targets, in a parse-only CI setup - `row_number() over (…)` windows are read from the whole file, so an `ORDER BY` on its own line is seen: the "dedup has no ORDER BY" rule no longer fires falsely on multi-line windows, and the load-timestamp rule sees them. A window counts only when an added line falls inside it Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/review/dbt-patterns.ts | 44 ++++++++++++---- .../src/altimate/review/orchestrate.ts | 46 +++++++++++----- .../review-incremental-semantics.test.ts | 52 +++++++++++++++++++ 3 files changed, 120 insertions(+), 22 deletions(-) diff --git a/packages/opencode/src/altimate/review/dbt-patterns.ts b/packages/opencode/src/altimate/review/dbt-patterns.ts index 91053d7515..f96eca2649 100644 --- a/packages/opencode/src/altimate/review/dbt-patterns.ts +++ b/packages/opencode/src/altimate/review/dbt-patterns.ts @@ -225,15 +225,42 @@ const detectNotIn: Detector = (c) => { }) } +/** + * `row_number() over (…)` windows in the model, read from the whole file so a + * window split across lines is seen whole. A window counts as part of the + * change when one of the PR's added lines falls inside it. + */ +function dedupWindows(c: Ctx): Array<{ spec: string; text: string }> { + const sql = stripComments(c.newSql) + const added = c.added.map((l) => stripComments(l).trim()).filter((l) => l.length >= 3) + const out: Array<{ spec: string; text: string }> = [] + for (const m of sql.matchAll(/row_number\s*\(\s*\)\s*over\s*\(/gi)) { + const open = m.index! + m[0].length - 1 + let depth = 0 + let close = -1 + for (let i = open; i < sql.length; i++) { + if (sql[i] === "(") depth++ + else if (sql[i] === ")" && --depth === 0) { + close = i + break + } + } + if (close < 0) continue + const text = sql.slice(m.index!, close + 1) + if (!added.some((l) => text.includes(l) || l.includes(text.replace(/\s+/g, " ")))) continue + out.push({ spec: sql.slice(open + 1, close).replace(/\s+/g, " ").trim(), text: text.replace(/\s+/g, " ") }) + } + return out +} + // 9. Dedup via ROW_NUMBER()/QUALIFY whose ORDER BY has no unique tiebreaker. const detectDedupTie: Detector = (c) => { - const hit = c.added.find((l) => /row_number\s*\(\s*\)\s*over\s*\(/i.test(stripComments(l))) - if (!hit) return null // Only flag dedup with NO ORDER BY — that is unambiguously non-deterministic. // A present ORDER BY (even single-column) is the developer's deterministic // choice and the dbt-recommended pattern; warning "it might not be unique" // on every dedup is noise (false positives on the common, correct case). - if (/order\s+by/i.test(stripComments(hit))) return null + const hit = dedupWindows(c).find((w) => !/order\s+by/i.test(w.spec)) + if (!hit) return null return makeFinding({ severity: "warning", category: "dedup", @@ -244,7 +271,7 @@ const detectDedupTie: Detector = (c) => { file: c.file.path, model: c.model, confidence: "high", - evidence: { tool: "dbt-patterns", result: { rule: "dedup_no_tiebreaker", line: hit.trim() } }, + evidence: { tool: "dbt-patterns", result: { rule: "dedup_no_tiebreaker", line: hit.text } }, ruleKey: "dedup:no-tiebreaker", }) } @@ -257,11 +284,8 @@ const detectDedupTie: Detector = (c) => { const LOAD_TS_RE = /^(_?(ingest(ion|ed)?|load(ed)?|etl|extract(ed)?|batch)(_at|_ts|_time|_timestamp|_date|_dt)|_?(ingestion|load|etl|batch)_timestamp|_loaded_at)$/i const detectDedupLoadTimestamp: Detector = (c) => { - for (const line of c.added) { - const s = stripComments(line) - const m = /row_number\s*\(\s*\)\s*over\s*\(([^)]*)\)/i.exec(s) - if (!m) continue - const order = /order\s+by\s+(.+)$/i.exec(m[1])?.[1] + for (const w of dedupWindows(c)) { + const order = /order\s+by\s+(.+)$/i.exec(w.spec)?.[1] if (!order) continue const keys = order.split(",").map((k) => k.trim().replace(/\s+(asc|desc)\b.*$/i, "").replace(/\s+nulls\s+(first|last)$/i, "")) if (keys.length !== 1) continue @@ -277,7 +301,7 @@ const detectDedupLoadTimestamp: Detector = (c) => { file: c.file.path, model: c.model, confidence: "high", - evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: line.trim() } }, + evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: w.text } }, ruleKey: "dedup:load-timestamp-only", }) } diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index aadb9fe26e..58566815df 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -648,19 +648,41 @@ async function columnDelta( dialect: string, ): Promise<{ added: string[]; removed: string[]; current: Set }> { const none = { added: [], removed: [], current: new Set() } - const { file, engineNewSql, engineOldSql } = ctx - if (!engineNewSql || file.status === "deleted") return none - const current = await outputColumns(runner, engineNewSql, dialect) - if (!current.size) return none - if (!hasOldSide(file)) return { added: [...current], removed: [], current } - if (!engineOldSql) return none - const before = await outputColumns(runner, engineOldSql, dialect) - if (!before.size) return none - return { - added: [...current].filter((c) => !before.has(c)), - removed: [...before].filter((c) => !current.has(c)), - current, + const { file, engineNewSql, engineOldSql, newSql, oldSql } = ctx + if (file.status === "deleted") return none + const withOld = hasOldSide(file) + const delta = (current: Set, before?: Set) => + !current.size || (withOld && !before?.size) + ? none + : { + added: [...current].filter((c) => !before?.has(c)), + removed: before ? [...before].filter((c) => !current.has(c)) : [], + current, + } + // Compiled SQL on both sides: column lineage plus the final SELECT's names. + if (engineNewSql && (!withOld || engineOldSql)) { + const current = await outputColumns(runner, engineNewSql, dialect) + const before = withOld ? await outputColumns(runner, engineOldSql!, dialect) : undefined + return delta(current, before) } + // No compiled SQL (CI that only runs `dbt parse`): the raw model's own SELECT + // list, when it names its columns. Columns produced inside a macro stay unknown. + if (!newSql) return none + const current = rawSelectList(newSql) + const before = withOld && oldSql ? rawSelectList(oldSql) : undefined + return delta(current, before) +} + +/** Output names of a raw (Jinja) model's final SELECT; empty when a Jinja expression is in the list. */ +function rawSelectList(sql: string): Set { + const plain = sql + .replace(/\{#[\s\S]*?#\}/g, " ") + .replace(/\{%[\s\S]*?%\}/g, " ") + .replace(/\{\{[\s\S]*?\}\}/g, " __jinja__ ") + const names = selectListNames(plain) + if (!names || names.includes("__jinja__")) return new Set() + // A Jinja expression inside an item (e.g. `{{ col }} as x`) leaves its alias intact. + return new Set(names) } const ticks = (xs: string[]) => xs.map((x) => `\`${x}\``).join(", ") diff --git a/packages/opencode/test/altimate/review-incremental-semantics.test.ts b/packages/opencode/test/altimate/review-incremental-semantics.test.ts index 0766b1acb8..98bd94029f 100644 --- a/packages/opencode/test/altimate/review-incremental-semantics.test.ts +++ b/packages/opencode/test/altimate/review-incremental-semantics.test.ts @@ -254,3 +254,55 @@ describe("end to end through the orchestrator", () => { expect(env.verdict).not.toBe("REQUEST_CHANGES") }) }) + +describe("dedup windows split across lines", () => { + const sql = "select *,\n row_number() over (\n partition by job_id\n order by ingestion_timestamp desc\n ) as rn\nfrom t" + const rulesFor = (diffAdded: string) => + detectModelPatterns({ path: "models/m.sql", status: "modified", diff: diffAdded }, sql, DEFAULT_RUBRIC).map( + (f) => (f.evidence?.result as { rule?: string })?.rule, + ) + test("an ORDER BY on its own line is seen: no false 'no ORDER BY', and the load-timestamp rule fires", () => { + const r = rulesFor("+ row_number() over (\n") + expect(r).not.toContain("dedup_no_tiebreaker") + expect(r).toContain("dedup_load_timestamp_only") + }) + test("a window the PR doesn't touch is not reported", () => { + expect(rulesFor("+select 1 as unrelated\n")).not.toContain("dedup_load_timestamp_only") + }) +}) + +describe("column changes without compiled SQL (dbt parse-only CI)", () => { + test("on_schema_change='fail' sees columns added to the raw model's SELECT", async () => { + const cfg = "{{ config(materialized='incremental', unique_key='id', on_schema_change='fail') }}\n" + const oldSql = `${cfg}select id, amount from {{ ref('src') }}` + const newSql = `${cfg}select id, amount, region from {{ ref('src') }}` + const env = await runReview({ + changedFiles: [{ path: "models/marts/fct.sql", status: "modified", diff: `-${oldSql}\n+${newSql}\n` }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["dbt_patterns"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return {} + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: false } + }, + async detectPii() { + return { columns: [] } + }, + async columnLineage() { + return [] + }, + } as ReviewRunner, + getContent: async (_f, side) => (side === "old" ? oldSql : newSql), + }) + expect(env.findings.find((f) => f.title.includes("on_schema_change"))?.body).toContain("adds `region`") + }) +}) From 7da6850781135051a1e298d4cc73a467f0dc97a9 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 13 Nov 2026 13:32:13 +0530 Subject: [PATCH 3/3] feat(review): data tests that can't do their job: self-comparisons, inherited warn/thresholds, loosened project thresholds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - a test added or changed in the PR that compares a model with itself (`dbt_utils.equality`/`equal_rowcount`/… against its own `ref`, or a `relationships` test to its own column) passes by construction (warning) - new tests that inherit `severity: warn` or an `error_if` threshold from `dbt_project.yml` (walked from the top through the project name and folders, deeper wins) are reported once per yml (suggestion) - a `dbt_project.yml` change that turns a folder's tests to `warn` or sets an `error_if` threshold is reported (warning) - reads dbt 1.8+ `arguments:` too; all findings are catalog origin (never block) Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 8 + .../src/altimate/review/orchestrate.ts | 11 + .../src/altimate/review/test-adequacy.ts | 214 ++++++++++++++++++ .../altimate/review-test-adequacy.test.ts | 83 +++++++ 4 files changed, 316 insertions(+) create mode 100644 packages/opencode/src/altimate/review/test-adequacy.ts create mode 100644 packages/opencode/test/altimate/review-test-adequacy.test.ts diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index 4c344ee9ca..264fae95c5 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -505,6 +505,14 @@ The deterministic rules group into four families: - **dbt unit tests.** A unit test whose `given` inputs are all `rows: []` is reported: it either never tests the model's logic, or expects rows the model has nothing to build from. +- **Data tests that can't do their job.** These are checked on tests added or + changed in the PR: + - a test that compares a model with itself (`dbt_utils.equality` or + `equal_rowcount` against its own `ref`, or a `relationships` test pointing + at its own column) passes by construction; + - new tests that inherit `severity: warn` or an `error_if` threshold from + `dbt_project.yml` report failures without failing the build; + - a `dbt_project.yml` change that loosens those thresholds for a whole folder. - **Incremental and snapshot semantics.** Config is read from the manifest (project, yml and inline, as dbt merges them) with the head's inline `{{ config() }}` on top. Formatting-only edits never count as a change. diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 58566815df..a62a08dade 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -17,6 +17,7 @@ import { detectModelPatterns, detectSchemaYmlPatterns, splitDiff } from "./dbt-p import { listMirrorsModel, type ConsumerColumnList } from "./consumer-lists" import { detectUnitTestFixtures } from "./unit-tests" import { selectListNames } from "./select-list" +import { detectTestAdequacy, detectThresholdLoosening } from "./test-adequacy" import { incrementalFindings, snapshotFindings, snapshotYmlFindings } from "./incremental-semantics" import { type AiLaneStatus, type AiReviewInput, type AiReviewResult } from "./ai-review" @@ -1665,6 +1666,9 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "schema_yml") + // The head's dbt_project.yml (test thresholds inherited by new tests), read once. + const projectYmlPath = `${input.layout?.root ? `${input.layout.root.replace(/\/$/, "")}/` : ""}dbt_project.yml` + const projectYml = schemaFiles.length ? await getContent?.(projectYmlPath, "new").catch(() => undefined) : undefined if (schemaFiles.length) { const schemaFindingSets = await Promise.all( schemaFiles.map(async (file) => { @@ -1677,11 +1681,18 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "project_config" && /(^|\/)dbt_project\.ya?ml$/.test(f.path))) { + if (file.status !== "modified") continue + const [oldContent, newContent] = await Promise.all([getContent?.(file.path, "old"), getContent?.(file.path, "new")]) + all.push(detectThresholdLoosening(file, oldContent, newContent)) + } // Architectural dedup: the regex `dbt-patterns`/`rule-catalog` layer is a // diff-aware, Jinja-aware, offline fallback. For the STRUCTURAL SQL checks diff --git a/packages/opencode/src/altimate/review/test-adequacy.ts b/packages/opencode/src/altimate/review/test-adequacy.ts new file mode 100644 index 0000000000..65fb2848cc --- /dev/null +++ b/packages/opencode/src/altimate/review/test-adequacy.ts @@ -0,0 +1,214 @@ +import YAML from "yaml" +import { makeFinding, type Finding } from "./finding" +import type { ChangedFile, DbtProjectLayout } from "./diff-filter" + +/** + * Data tests that cannot do their job, found from yml alone (any adapter): + * - a test that compares a model with itself (`dbt_utils.equality` / + * `equal_rowcount` against its own `ref`, or a `relationships` test pointing + * at its own column) passes by construction; + * - tests added in the PR that inherit `severity: warn` (or an `error_if` + * threshold) from `dbt_project.yml` report failures without failing the build; + * - a PR that loosens those thresholds in `dbt_project.yml` weakens every test + * under that path. + * Only tests added or changed in the PR are checked. + */ + +interface TestRef { + entity: string + column?: string + test: string + args: Record +} + +const SELF_COMPARE = new Set([ + "equality", + "equal_rowcount", + "cardinality_equality", + "fewer_rows_than", + "expect_table_row_count_to_equal_other_table", + "expect_table_aggregation_to_equal_other_table", +]) + +function parse(text?: string): any { + if (!text) return undefined + try { + return YAML.parse(text) + } catch { + return undefined + } +} + +function testsOf(doc: any): TestRef[] { + const out: TestRef[] = [] + const read = (list: unknown, entity: string, column?: string) => { + if (!Array.isArray(list)) return + for (const t of list) { + if (typeof t === "string") out.push({ entity, column, test: t, args: {} }) + else if (t && typeof t === "object") { + const [k] = Object.keys(t) + if (k) out.push({ entity, column, test: k, args: (t as any)[k] ?? {} }) + } + } + } + for (const group of ["models", "snapshots", "seeds"]) { + for (const e of Array.isArray(doc?.[group]) ? doc[group] : []) { + if (typeof e?.name !== "string") continue + read(e.data_tests ?? e.tests, e.name) + for (const c of Array.isArray(e.columns) ? e.columns : []) + if (typeof c?.name === "string") read(c.data_tests ?? c.tests, e.name, c.name) + } + } + return out +} + +const key = (t: TestRef) => `${t.entity}|${t.column ?? ""}|${t.test}|${JSON.stringify(t.args)}` +const arg = (t: TestRef, name: string) => t.args?.[name] ?? t.args?.arguments?.[name] +const shortName = (test: string) => test.split(".").pop()!.toLowerCase() +const refsEntity = (v: unknown, entity: string) => + typeof v === "string" && new RegExp(`ref\\(\\s*['"]${entity.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}['"]\\s*\\)`).test(v) + +function finding(file: ChangedFile, model: string | undefined, rule: string, severity: Finding["severity"], title: string, body: string, extra = ""): Finding { + return makeFinding({ + severity, + category: "test_coverage", + title, + body, + file: file.path, + model, + confidence: "high", + evidence: { tool: "test-adequacy", result: { rule } }, + ruleKey: `test_coverage:${rule}${extra}`, + }) +} + +/** `+severity` / `+error_if` / `+warn_if` (with or without `+`) on one config node. */ +function thresholds(node: any): { severity?: string; error_if?: string; warn_if?: string } { + if (!node || typeof node !== "object") return {} + const get = (k: string) => (node[`+${k}`] ?? node[k]) as unknown + const str = (v: unknown) => (typeof v === "string" ? v.trim().toLowerCase() : undefined) + return { severity: str(get("severity")), error_if: str(get("error_if")), warn_if: str(get("warn_if")) } +} + +/** + * Effective test thresholds for a yml under `segments` (folders below the model + * path), walking `data_tests:` (or `tests:`) from the top, through the project + * name, down the folders; deeper levels override. `path` is where the effective + * value was set. + */ +export function inheritedThresholds(projectYml: any, segments: string[]): { severity?: string; error_if?: string; path: string[] } { + const tree = projectYml?.data_tests ?? projectYml?.tests + if (!tree || typeof tree !== "object") return { path: [] } + const name = projectYml?.name + const chain = name && tree[name] && typeof tree[name] === "object" ? [String(name), ...segments] : segments + const levels: Array<[string[], any]> = [[[], tree]] + let node = tree + for (let i = 0; i < chain.length; i++) { + node = node?.[chain[i]] + if (!node || typeof node !== "object") break + levels.push([chain.slice(0, i + 1), node]) + } + const eff: { severity?: string; error_if?: string } = {} + let path: string[] = [] + for (const [at, n] of levels) { + const t = thresholds(n) + if (t.severity) eff.severity = t.severity + if (t.error_if) eff.error_if = t.error_if + if (t.severity || t.error_if) path = at.length ? at : ["(all tests)"] + } + return { ...eff, path } +} + +/** Folders of a yml below its model path (e.g. `models/marts/x.yml` → `["marts"]`). */ +function segmentsOf(file: string, layout?: DbtProjectLayout): string[] { + const root = layout?.root ? layout.root.replace(/\/$/, "") + "/" : "" + const rel = root && file.startsWith(root) ? file.slice(root.length) : file + const parts = rel.split("/").slice(0, -1) + const modelDirs = layout?.dirs.model ?? ["models"] + return modelDirs.includes(parts[0]) ? parts.slice(1) : parts +} + +export function detectTestAdequacy(input: { + file: ChangedFile + oldContent?: string + newContent?: string + projectYml?: string + layout?: DbtProjectLayout +}): Finding[] { + const { file } = input + if (file.status === "deleted") return [] + const newDoc = parse(input.newContent) + if (!newDoc) return [] + const before = new Set(testsOf(parse(input.oldContent)).map(key)) + const added = testsOf(newDoc).filter((t) => !before.has(key(t))) + const out: Finding[] = [] + + for (const t of added) { + const name = shortName(t.test) + const self = + (SELF_COMPARE.has(name) && ["compare_model", "to", "compare_to"].some((a) => refsEntity(arg(t, a), t.entity))) || + (name === "relationships" && refsEntity(arg(t, "to"), t.entity) && String(arg(t, "field") ?? "").toLowerCase() === (t.column ?? "").toLowerCase()) + if (self) + out.push( + finding(file, t.entity, "test-compares-model-with-itself", "warning", `${t.entity}: \`${t.test}\` compares the model with itself`, + `This test checks \`${t.entity}\`${t.column ? `.\`${t.column}\`` : ""} against \`ref('${t.entity}')\`, so it passes by construction. ` + + "Point it at the independent source of truth it is meant to reconcile against.", `:${key(t)}`), + ) + } + + const project = parse(input.projectYml) + if (project && added.length) { + const inherited = inheritedThresholds(project, segmentsOf(file.path, input.layout)) + const own = (t: TestRef) => { + const c = t.args?.config ?? {} + return [c.severity, c.error_if, c.warn_if, t.args?.severity, t.args?.error_if].some((v) => v !== undefined) + } + const loose = added.filter((t) => !own(t)) + const where = inherited.path.length ? ` (\`data_tests:\` → ${inherited.path.map((p) => `\`${p}\``).join(" → ")})` : "" + if (loose.length && inherited.severity === "warn") + out.push( + finding(file, undefined, "test-inherits-warn", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`severity: warn\``, + `They inherit \`severity: warn\` from \`dbt_project.yml\`${where}, so failures are reported but never fail \`dbt build\`. ` + + "If a test should gate the build, set `config: {severity: error}` on it."), + ) + else if (loose.length && inherited.error_if && inherited.error_if !== "!= 0") + out.push( + finding(file, undefined, "test-inherits-threshold", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`error_if: ${inherited.error_if}\``, + `They fail only when \`${inherited.error_if}\` rows fail, a threshold inherited from \`dbt_project.yml\`${where}. ` + + "Set `config: {error_if: '!= 0'}` on tests that must catch any bad row."), + ) + } + return out +} + +/** A `dbt_project.yml` change that loosens test thresholds for a whole path. */ +export function detectThresholdLoosening(file: ChangedFile, oldContent?: string, newContent?: string): Finding[] { + const oldDoc = parse(oldContent) + const newDoc = parse(newContent) + if (!oldDoc || !newDoc) return [] + const flat = (doc: any) => { + const out = new Map>() + const visit = (node: any, path: string[]) => { + if (!node || typeof node !== "object") return + out.set(path.join(" → ") || "(all tests)", thresholds(node)) + for (const [k, v] of Object.entries(node)) if (!k.startsWith("+") && v && typeof v === "object") visit(v, [...path, k]) + } + visit(doc?.data_tests ?? doc?.tests, []) + return out + } + const before = flat(oldDoc) + const out: Finding[] = [] + for (const [path, t] of flat(newDoc)) { + const prev = before.get(path) ?? {} + const toWarn = t.severity === "warn" && prev.severity !== "warn" + const newThreshold = t.error_if && t.error_if !== "!= 0" && t.error_if !== prev.error_if + if (!toWarn && !newThreshold) continue + out.push( + finding(file, undefined, "test-thresholds-loosened", "warning", `test thresholds loosened for \`${path}\``, + (toWarn ? "`severity: warn` now applies" : `\`error_if: ${t.error_if}\` now applies`) + + ` to every data test under \`${path}\` in \`dbt_project.yml\`, so those tests stop failing the build. ` + + "Prefer setting it on the specific tests that need it.", `:${path}`), + ) + } + return out +} diff --git a/packages/opencode/test/altimate/review-test-adequacy.test.ts b/packages/opencode/test/altimate/review-test-adequacy.test.ts new file mode 100644 index 0000000000..644e664d14 --- /dev/null +++ b/packages/opencode/test/altimate/review-test-adequacy.test.ts @@ -0,0 +1,83 @@ +import { describe, test, expect } from "bun:test" +import YAML from "yaml" +import { detectTestAdequacy, detectThresholdLoosening, inheritedThresholds } from "../../src/altimate/review/test-adequacy" + +/** Data tests that can't do their job, from yml alone (any adapter). */ + +const file = { path: "models/marts/_fct.yml", status: "modified" as const } +const rule = (fs: { evidence?: { result?: unknown } }[]) => fs.map((f) => (f.evidence?.result as { rule: string }).rule) + +describe("tests that compare a model with itself", () => { + const yml = (compare: string) => `models: + - name: fct_orders + data_tests: + - dbt_utils.equal_rowcount: + compare_model: ${compare} + columns: + - name: customer_id + data_tests: + - relationships: + to: ref('fct_orders') + field: customer_id +` + test("own-ref comparisons and self relationships pass by construction", () => { + const out = detectTestAdequacy({ file, newContent: yml("ref('fct_orders')") }) + expect(rule(out)).toEqual(["test-compares-model-with-itself", "test-compares-model-with-itself"]) + }) + + test("a comparison with another model is fine; unchanged tests are skipped", () => { + const other = yml("ref('stg_orders')").replace("to: ref('fct_orders')", "to: ref('dim_customers')") + expect(detectTestAdequacy({ file, newContent: other })).toEqual([]) + expect(detectTestAdequacy({ file, oldContent: yml("ref('fct_orders')"), newContent: yml("ref('fct_orders')") })).toEqual([]) + }) + + test("dbt 1.8+ `arguments:` form is read too", () => { + const y = `models:\n - name: m\n data_tests:\n - dbt_utils.equality:\n arguments:\n compare_model: ref("m")\n` + expect(rule(detectTestAdequacy({ file: { path: "models/_m.yml", status: "added" }, newContent: y }))).toEqual(["test-compares-model-with-itself"]) + }) +}) + +describe("thresholds inherited from dbt_project.yml", () => { + const project = `name: shop +data_tests: + shop: + marts: + +severity: warn + staging: + +error_if: ">100" +` + const newTests = (own = "") => `models: + - name: fct_orders + columns: + - name: id + data_tests: + - not_null${own} +` + test("the effective value walks project → folders, deeper wins", () => { + const doc = YAML.parse(project) + expect(inheritedThresholds(doc, ["marts"])).toEqual({ severity: "warn", path: ["shop", "marts"] }) + expect(inheritedThresholds(doc, ["staging", "x"])).toEqual({ error_if: ">100", path: ["shop", "staging"] }) + expect(inheritedThresholds(doc, ["core"])).toEqual({ path: [] }) + }) + + test("new tests under a warn folder are reported; a test with its own severity is not", () => { + const out = detectTestAdequacy({ file, newContent: newTests(), projectYml: project }) + expect(rule(out)).toEqual(["test-inherits-warn"]) + expect(out[0].body).toContain("`shop` → `marts`") + const own = newTests(":\n config:\n severity: error").replace("- not_null:", "- not_null:") + expect(detectTestAdequacy({ file, newContent: own, projectYml: project })).toEqual([]) + }) + + test("an inherited error_if threshold is reported", () => { + const out = detectTestAdequacy({ file: { path: "models/staging/_stg.yml", status: "modified" }, newContent: newTests(), projectYml: project }) + expect(rule(out)).toEqual(["test-inherits-threshold"]) + }) + + test("a dbt_project.yml change that loosens thresholds is reported", () => { + const before = "name: shop\ndata_tests:\n shop:\n marts:\n +severity: error\n" + const after = "name: shop\ndata_tests:\n shop:\n marts:\n +severity: warn\n" + const out = detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, before, after) + expect(out.map((f) => f.title)).toEqual(["test thresholds loosened for `shop → marts`"]) + expect(detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, after, after)).toEqual([]) + }) +})