Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions docs/docs/usage/dbt-pr-review.md
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ riskTierPathTokens: # OFF by default — see "Risk-tier path tokens" b
pii: [ssn, email, phone] # or supply your own case-insensitive tokens
allowForceTier: true # set false in gate CI: --force-tier is then rejected
checkContractColumns: false # opt-in: new columns missing from an enforced contract (CI that doesn't run `dbt build`)
incrementalSliceVars: [] # opt-in: vars that pick one slice per incremental run (e.g. [tenant])
consumerColumnLists: [] # OFF by default — see "Consumers outside dbt" below
rubric:
blockOn: [lineage_breakage, contract_violation, pii_exposure, semantic_change]
Expand Down Expand Up @@ -486,6 +487,31 @@ The deterministic rules group into four families:
- **dbt unit tests.** A unit test whose `given` inputs are all `rows: []` is
reported: it either never tests the model's logic, or expects rows the model has nothing to
build from.
- **Incremental and snapshot semantics.** Config is read from the manifest
(project, yml and inline, as dbt merges them) with the head's inline
`{{ config() }}` on top. Formatting-only edits never count as a change.
- an existing incremental model gains a column, or its `is_incremental()`
filter changes, and the PR doesn't say how existing rows are handled
(`--full-refresh`, a backfill, or forward-only) — a suggestion;
- `partition_by` changes on an incremental model (needs a full refresh);
- `on_schema_change='fail'` and the change adds or removes columns, so the
first incremental run after merge fails;
- a snapshot's `check_cols` or `strategy` changes (`.sql` or dbt 1.9+ yml
snapshots), which can version every row on the next run;
- `row_number()` dedup ordered only by a load timestamp (`loaded_at`,
`ingestion_timestamp`) — a suggestion; loader sync columns such as
`_fivetran_synced` are not flagged.
- **Opt-in: per-slice incremental runs.** For projects that run one slice per
invocation (a tenant or region chosen by a var), list those vars:
`incrementalSliceVars: [tenant]`. A `delete+insert` or `merge` model filtered
by one of them must include the filtered column in `unique_key`, or one
slice's run deletes or overwrites another's rows.
- **Adapter-dependent advice** fires only where the adapter table establishes
the behavior. On Trino, `on_table_exists='replace'` with an enforced contract
runs `create or replace` then `insert`, and `drop` removes the table during
the build, so readers outside dbt can see an empty or missing table (a
suggestion). `unique_key` and strategy changes are covered by the engine's
`DBT011`/`DBT012`.
- **dbt config / Jinja (`DBT0xx`).** A core **minijinja** parser reads
`{{ config(...) }}` and the model body: incremental model without an
`is_incremental()` guard, `merge`/`delete+insert` without a `unique_key`,
Expand Down
16 changes: 15 additions & 1 deletion packages/opencode/src/altimate/review/adapter-capabilities.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,13 @@
export interface AdapterCapability {
/** `int / int` returns an integer, dropping the fraction (7 / 2 = 3). */
integerDivisionTruncates: boolean
/**
* A `table` model with an enforced contract and `on_table_exists='replace'`
* is replaced atomically. False = `create or replace` then `insert` as two
* statements (readers see an empty table), and `on_table_exists='drop'`
* leaves no table during the build. Undefined = not established.
*/
contractTableReplaceAtomic?: boolean
/** Documentation backing the entries above. */
sources: string[]
}
Expand All @@ -26,7 +33,14 @@ export const ADAPTER_CAPABILITIES: Record<string, AdapterCapability> = {
},
trino: {
integerDivisionTruncates: true,
sources: ["https://trino.io/docs/current/functions/math.html"],
contractTableReplaceAtomic: false,
sources: [
"https://trino.io/docs/current/functions/math.html",
// create_table_as with an enforced contract: `create or replace table` then `insert`;
// on_table_exists='drop' drops the table before building it.
"https://github.com/starburstdata/dbt-trino/blob/master/dbt/include/trino/macros/adapters.sql",
"https://github.com/starburstdata/dbt-trino/blob/master/dbt/include/trino/macros/materializations/table.sql",
],
},
athena: {
// Athena SQL runs on Trino.
Expand Down
6 changes: 6 additions & 0 deletions packages/opencode/src/altimate/review/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ export const ReviewConfig = z.object({
* whose first capture group is the list, `model` the model it mirrors
* (omitted = files that mention the model by name).
*/
/**
* Opt-in: vars that select the slice one incremental run processes (e.g.
* `tenant`, `region`). A `delete+insert`/`merge` model filtered by one of
* them must include the filtered column in `unique_key`.
*/
incrementalSliceVars: z.array(z.string()).default([]),
consumerColumnLists: z
.array(z.object({ files: z.string(), pattern: z.string(), model: z.string().optional() }))
.default([]),
Expand Down
36 changes: 36 additions & 0 deletions packages/opencode/src/altimate/review/dbt-patterns.ts
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,41 @@ const detectDedupTie: Detector = (c) => {
})
}

// 9b. Dedup ordered ONLY by a load/ingestion timestamp. Rows loaded in one batch
// share that value, so which version survives is arbitrary (SCD2 versions ranked
// by `ingestion_timestamp` alone). A business column (`updated_at`) stays
// silent, and so do loader sync columns (`_fivetran_synced`, `_airbyte_*`):
// ordering by them is the standard staging idiom and ties are rare per key.
const LOAD_TS_RE =
/^(_?(ingest(ion|ed)?|load(ed)?|etl|extract(ed)?|batch)(_at|_ts|_time|_timestamp|_date|_dt)|_?(ingestion|load|etl|batch)_timestamp|_loaded_at)$/i
const detectDedupLoadTimestamp: Detector = (c) => {
for (const line of c.added) {
const s = stripComments(line)
const m = /row_number\s*\(\s*\)\s*over\s*\(([^)]*)\)/i.exec(s)
if (!m) continue
const order = /order\s+by\s+(.+)$/i.exec(m[1])?.[1]
if (!order) continue
const keys = order.split(",").map((k) => k.trim().replace(/\s+(asc|desc)\b.*$/i, "").replace(/\s+nulls\s+(first|last)$/i, ""))
if (keys.length !== 1) continue
const col = keys[0].split(".").pop()!.replace(/["`]/g, "")
if (!LOAD_TS_RE.test(col)) continue
return makeFinding({
severity: "suggestion",
category: "dedup",
title: `${c.model}: dedup ordered only by load timestamp \`${col}\``,
body:
`\`row_number()\` is ordered only by \`${col}\`. Rows loaded in the same batch share it, so which version survives is ` +
"arbitrary and can change between runs. Order by the source's own change time first and add a unique tie-breaker.",
file: c.file.path,
model: c.model,
confidence: "high",
evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: line.trim() } },
ruleKey: "dedup:load-timestamp-only",
})
}
return null
}

// 11. Function/cast wrapped around a column in an added WHERE — defeats partition pruning.
const detectPartitionFunction: Detector = (c) => {
const hit = c.added.find((l) => {
Expand Down Expand Up @@ -735,6 +770,7 @@ const MODEL_DETECTORS: Detector[] = [
detectFanout,
detectNotIn,
detectDedupTie,
detectDedupLoadTimestamp,
detectPartitionFunction,
detectCountDistinct,
detectPiiIntoMart,
Expand Down
Loading
Loading