Repository navigation
fix(review): review the PR's own changes: real base, merge-base diff, project scope, renames, model-paths, no raw Jinja #1719
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ import { promises as fs } from "node:fs" | |
| import path from "node:path" | ||
| import YAML from "yaml" | ||
| import { safeReadInside } from "./git" | ||
| import { DEFAULT_DBT_DIRS, type DbtProjectLayout } from "./diff-filter" | ||
|
|
||
| /** | ||
| * Resolve dbt's COMPILED SQL for static analysis. | ||
|
|
@@ -36,6 +37,39 @@ export async function dbtProjectName(cwd: string): Promise<string | undefined> { | |
| return undefined | ||
| } | ||
|
|
||
| const LAYOUT_KEYS: Record<keyof DbtProjectLayout["dirs"], string> = { | ||
| model: "model-paths", | ||
| snapshot: "snapshot-paths", | ||
| seed: "seed-paths", | ||
| macro: "macro-paths", | ||
| test: "test-paths", | ||
| analysis: "analysis-paths", | ||
| } | ||
|
|
||
| /** | ||
| * Read the project's directory layout (`model-paths`, `snapshot-paths`, …) | ||
| * from dbt_project.yml, falling back to dbt's defaults per key. `root` is the | ||
| * project directory relative to the repository root. Undefined when there is | ||
| * no readable dbt_project.yml in `projectDir`. | ||
| */ | ||
| export async function dbtProjectLayout(projectDir: string, root: string): Promise<DbtProjectLayout | undefined> { | ||
| for (const f of ["dbt_project.yml", "dbt_project.yaml"]) { | ||
| let doc: any | ||
| try { | ||
| doc = YAML.parse(await fs.readFile(path.join(projectDir, f), "utf8")) | ||
| } catch { | ||
| continue | ||
| } | ||
| const dirs = { ...DEFAULT_DBT_DIRS } | ||
| for (const [role, key] of Object.entries(LAYOUT_KEYS) as Array<[keyof typeof dirs, string]>) { | ||
| const v = doc?.[key] | ||
| if (Array.isArray(v) && v.length && v.every((x) => typeof x === "string")) dirs[role] = v | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: This records Jinja path expressions without rendering them. Projects using Prompt for AI agents |
||
| } | ||
| return { root: root.replace(/\\/g, "/"), dirs } | ||
| } | ||
| return undefined | ||
| } | ||
|
|
||
| export interface CompiledResolverOptions { | ||
| /** dbt project root (the dir containing `dbt_project.yml`). */ | ||
| cwd: string | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -55,23 +55,85 @@ export type DbtFileKind = | |
| | "project_config" // dbt_project.yml / profiles.yml / packages.yml | ||
| | "other" | ||
|
|
||
| /** Classify a changed file by its role in a dbt project. */ | ||
| export function classifyDbtFile(path: string): DbtFileKind { | ||
| const p = path.replace(/\\/g, "/").toLowerCase() | ||
| type DbtRole = "model" | "snapshot" | "seed" | "macro" | "test" | "analysis" | ||
|
|
||
| /** | ||
| * Where a dbt project keeps each kind of file, from `dbt_project.yml` | ||
| * (`model-paths`, `snapshot-paths`, …). `root` is the project directory | ||
| * relative to the repository root ("" when the project is at the root). | ||
| */ | ||
| export interface DbtProjectLayout { | ||
| root: string | ||
| dirs: Record<DbtRole, string[]> | ||
| } | ||
|
|
||
| export const DEFAULT_DBT_DIRS: Record<DbtRole, string[]> = { | ||
| model: ["models"], | ||
| snapshot: ["snapshots"], | ||
| seed: ["seeds"], | ||
| macro: ["macros"], | ||
| test: ["tests"], | ||
| analysis: ["analyses"], | ||
| } | ||
|
|
||
| const norm = (p: string) => p.replace(/\\/g, "/") | ||
|
|
||
| /** `p` relative to `root`, or undefined when it lies outside. Both POSIX, repo-relative. */ | ||
| export function relativeToRoot(p: string, root: string): string | undefined { | ||
| const r = norm(root).replace(/^\.\/?/, "").replace(/\/+$/, "") | ||
| const f = norm(p) | ||
| if (!r) return f | ||
| return f.startsWith(r + "/") ? f.slice(r.length + 1) : undefined | ||
| } | ||
|
|
||
| /** The role of a project-relative path from the configured directories; the | ||
| * longest matching directory wins so `models/tests/x.sql` is a model when | ||
| * `models` is a model path. */ | ||
| function roleFromLayout(rel: string, dirs: Record<DbtRole, string[]>): DbtRole | undefined { | ||
| const p = rel.toLowerCase() | ||
| let best: { role: DbtRole; len: number } | undefined | ||
| for (const [role, list] of Object.entries(dirs) as Array<[DbtRole, string[]]>) { | ||
| for (const d of list) { | ||
| const dir = norm(d).toLowerCase().replace(/^\.\/?/, "").replace(/\/+$/, "") | ||
| if (dir && p.startsWith(dir + "/") && (!best || dir.length > best.len)) best = { role, len: dir.length } | ||
| } | ||
| } | ||
| return best?.role | ||
| } | ||
|
|
||
| /** Without a layout, the OUTERMOST conventional directory decides the role, | ||
| * so `models/tests/x.sql` is a model and `tests/models/x.sql` a test. */ | ||
| function roleFromConvention(p: string): DbtRole | undefined { | ||
| const segments = p.toLowerCase().split("/").slice(0, -1) | ||
| for (const seg of segments) { | ||
| for (const [role, list] of Object.entries(DEFAULT_DBT_DIRS) as Array<[DbtRole, string[]]>) { | ||
| if (list.includes(seg)) return role | ||
| } | ||
| } | ||
| return undefined | ||
| } | ||
|
|
||
| /** Classify a changed file by its role in a dbt project. With a layout, the | ||
| * project's own `*-paths` decide (custom `model-paths` are honoured); | ||
| * without one, conventional directory names do. */ | ||
| export function classifyDbtFile(path: string, layout?: DbtProjectLayout): DbtFileKind { | ||
| const p = norm(path).toLowerCase() | ||
| const isYaml = p.endsWith(".yml") || p.endsWith(".yaml") | ||
| if (/(^|\/)(dbt_project|profiles|packages|dependencies)\.ya?ml$/.test(p)) return "project_config" | ||
| if (/(^|\/)macros\//.test(p)) return "macro" | ||
| const rel = layout ? relativeToRoot(norm(path), layout.root) : undefined | ||
| const role = rel !== undefined && layout ? roleFromLayout(rel, layout.dirs) : roleFromConvention(p) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: When a layout is present but a path lies outside its root, this fallback classifies it using repository-wide conventional directories. Cross-boundary renames are retained, so moving Prompt for AI agents |
||
| if (role === "macro") return "macro" | ||
| // Snapshots split by extension: `.sql` files are snapshot definitions | ||
| // (tier-forcing, catalog-scoped), YAML property files under `snapshots/` | ||
| // are schema files (declared tests + column metadata) and should route to | ||
| // `schema_yml` so the test-removal detector runs on them. | ||
| if (/(^|\/)snapshots\//.test(p) && p.endsWith(".sql")) return "snapshot" | ||
| if (/(^|\/)seeds\//.test(p) && p.endsWith(".csv")) return "seed" | ||
| if (/(^|\/)tests\//.test(p) && p.endsWith(".sql")) return "test" | ||
| if (/(^|\/)analyses\//.test(p)) return "analysis" | ||
| if (/(^|\/)models\//.test(p) && p.endsWith(".py")) return "python_model" | ||
| if (/(^|\/)models\//.test(p) && p.endsWith(".sql")) return "model_sql" | ||
| if (isYaml && /(^|\/)(models|snapshots|seeds|tests)\//.test(p)) return "schema_yml" | ||
| if (role === "snapshot" && p.endsWith(".sql")) return "snapshot" | ||
| if (role === "seed" && p.endsWith(".csv")) return "seed" | ||
| if (role === "test" && p.endsWith(".sql")) return "test" | ||
| if (role === "analysis") return "analysis" | ||
| if (role === "model" && p.endsWith(".py")) return "python_model" | ||
| if (role === "model" && p.endsWith(".sql")) return "model_sql" | ||
| if (isYaml && (role === "model" || role === "snapshot" || role === "seed" || role === "test")) return "schema_yml" | ||
| if (isYaml && /(^|\/)(_?schema|_?models|_?sources|sources|properties)\.ya?ml$/.test(p)) return "schema_yml" | ||
| return "other" | ||
| } | ||
|
|
@@ -84,6 +146,21 @@ export interface ChangedFile { | |
| diff?: string | ||
| /** Previous path for renames. */ | ||
| oldPath?: string | ||
| /** Rename similarity (0–100) from git; 100 means the content is unchanged. */ | ||
| similarity?: number | ||
| /** dbt role, set by filterChangedFiles from the project layout. */ | ||
| kind?: DbtFileKind | ||
| } | ||
|
|
||
| /** The file's dbt role: the layout-aware kind when filterChangedFiles set it. */ | ||
| export function fileKind(file: ChangedFile): DbtFileKind { | ||
| return file.kind ?? classifyDbtFile(file.path) | ||
| } | ||
|
|
||
| /** True when a file has a previous version to compare against: a modification, | ||
| * or a rename (whose old side lives at `oldPath`). */ | ||
| export function hasOldSide(file: ChangedFile): boolean { | ||
| return file.status === "modified" || (file.status === "renamed" && !!file.oldPath) | ||
| } | ||
|
|
||
| // Compile a glob (`**`, `*`, `?`) to an anchored RegExp with path semantics: | ||
|
|
@@ -119,14 +196,58 @@ function globToRegExp(glob: string): RegExp { | |
| export function filterChangedFiles( | ||
| files: ChangedFile[], | ||
| extraExcludeGlobs: string[] = [], | ||
| layout?: DbtProjectLayout, | ||
| ): Array<ChangedFile & { kind: DbtFileKind }> { | ||
| const excluders = extraExcludeGlobs.map(globToRegExp) | ||
| // Normalize to forward slashes so `/`-based glob semantics hold on Windows paths. | ||
| const norm = (p: string) => p.replace(/\\/g, "/") | ||
| return files | ||
| .filter((f) => shouldReview(f.path)) | ||
| .filter((f) => !excluders.some((re) => re.test(norm(f.path)))) | ||
| .map((f) => ({ ...f, kind: classifyDbtFile(f.path) })) | ||
| .map((f) => ({ ...f, kind: classifyDbtFile(f.path, layout) })) | ||
| } | ||
|
|
||
| export interface ScopeResult { | ||
| /** Files inside the dbt project, with unchanged moves removed. */ | ||
| inScope: ChangedFile[] | ||
| /** Changed files outside the dbt project root (counted, not reviewed). */ | ||
| outOfScope: number | ||
| /** Renames with identical content that keep the model name (a pure move). */ | ||
| movedUnchanged: number | ||
| } | ||
|
|
||
| /** | ||
| * Restrict the PR's changed files to the dbt project being reviewed. | ||
| * | ||
| * - Files outside `root` belong to another project or to non-dbt code: they | ||
| * are counted, never reviewed (a monorepo with several dbt projects must not | ||
| * review its siblings). | ||
| * - A rename at 100% similarity that keeps the file name is a pure move: dbt | ||
| * resolves `ref()` by name, so nothing about the model changed and | ||
| * re-reviewing it would only repeat pre-existing findings. A rename that | ||
| * changes the file name stays in scope — it renames the model and can break | ||
| * downstream refs. | ||
| */ | ||
| export function scopeToProject(files: ChangedFile[], root: string | undefined): ScopeResult { | ||
| let outOfScope = 0 | ||
| let movedUnchanged = 0 | ||
| const inScope: ChangedFile[] = [] | ||
| const base = (p: string) => norm(p).split("/").pop() | ||
| for (const f of files) { | ||
| const isInside = (p: string) => root === undefined || relativeToRoot(p, root) !== undefined | ||
| const inside = isInside(f.path) | ||
| // A rename moving a file into or out of the project still concerns it. | ||
| const oldInside = !!f.oldPath && isInside(f.oldPath) | ||
| if (!inside && !oldInside) { | ||
| outOfScope++ | ||
| continue | ||
| } | ||
| if (f.status === "renamed" && f.similarity === 100 && f.oldPath && base(f.oldPath) === base(f.path) && inside && oldInside) { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: Prompt for AI agentsThere was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: Skipping a pure move assumes the move cannot change behavior, but dbt resolves several things by path.
Consider dropping the move only when the source and destination resolve to the same path-scoped config, or keep it in scope for the config/risk lanes. |
||
| movedUnchanged++ | ||
| continue | ||
|
Comment on lines
+244
to
+246
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Keep moves that change dbt behavior in the review. A 100%-similar rename from
|
||
| } | ||
| inScope.push(f) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Represent a cross-project rename as a removal or addition. If
|
||
| } | ||
| return { inScope, outOfScope, movedUnchanged } | ||
| } | ||
|
|
||
| /** Heuristic: does a yml diff declare dbt `sources:`? Tolerates unified-diff | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P2: This layout reader drops the legacy
source-pathsanddata-pathsaliases that the repository still supports. Older projects using those keys fall back to the default directories, causing custom model files to miss model review; honor the aliases when selecting each layout list.Prompt for AI agents