diff --git a/packages/opencode/src/altimate/native/dbt/column-answer.ts b/packages/opencode/src/altimate/native/dbt/column-answer.ts index 525ad3d9eb..24fc6b1ebd 100644 --- a/packages/opencode/src/altimate/native/dbt/column-answer.ts +++ b/packages/opencode/src/altimate/native/dbt/column-answer.ts @@ -465,6 +465,6 @@ export function describeAnswer(a: ColumnAnswer): string { const c = a.counts const what = a.direction === "downstream" ? "Column impact" : "Column trace (upstream)" const reach = `${c.columns} column${c.columns === 1 ? "" : "s"} in ${c.models} model${c.models === 1 ? "" : "s"}, ${c.hops} hop${c.hops === 1 ? "" : "s"}, ${c.ends} end${c.ends === 1 ? "" : "s"}` - const tiers = `${c.may_carry_models} more of the ${c.model_level} ${a.direction} may carry it${c.not_traced_models ? `, ${c.not_traced_models} of them with no sign (not traced)` : ""}` + const tiers = `${c.may_carry_models} more of the ${c.model_level} ${a.direction} may carry it${c.not_traced_models ? `, ${c.not_traced_models} of them not traced (no sign either way)` : ""}` return `${what}: ${reach}; ${tiers}${a.truncated ? " (truncated)" : ""} (metadata.column_answer)` } diff --git a/packages/opencode/src/altimate/native/dbt/project-lineage.ts b/packages/opencode/src/altimate/native/dbt/project-lineage.ts index ba659ff50b..ab22c8dc07 100644 --- a/packages/opencode/src/altimate/native/dbt/project-lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/project-lineage.ts @@ -102,10 +102,139 @@ function engineMade(name: string, stripped: string): boolean { return ENGINE_NAME.test(name) && !new RegExp(`(? name.toLowerCase()) - return names.length > 0 && new Set(names).size === names.length && !names.some((name) => engineMade(name, stripped)) + return names.length > 0 && new Set(names).size === names.length +} + +const QUOTED = '"(?:[^"]|"")*"|`[^`]*`|\\[[^\\]]*\\]' +const unquoteName = (name: string) => name.replace(/^["`[]|["`\]]$/g, "").toLowerCase() + +/** + * The items of the SQL's outermost SELECT list (the first one outside parentheses, so after any WITH), split at + * top-level commas, from literal- and comment-stripped SQL; [] when there is none. + */ +export function selectItems(stripped: string): string[] { + // A quoted name is blanked whole, so `"from"` or `"a,b"` ends no item. + const text = stripped.replace(new RegExp(QUOTED, "g"), (q) => " ".repeat(q.length)) + let depth = 0 + let start = -1 + const items: string[] = [] + let from = 0 + for (let i = 0; i < text.length; i++) { + const ch = text[i]! + if (ch === "(") depth++ + else if (ch === ")") depth-- + if (depth !== 0) continue + const word = /^[a-z_]+/i.exec(text.slice(i))?.[0]?.toLowerCase() + const boundary = i === 0 || !/[\w$]/.test(text[i - 1]!) + if (start < 0) { + if (boundary && word === "select") { + start = i + 6 + // A whole word only: `allocation` and `distinct_users` are column names, not modifiers. + const lead = /^\s+(?:distinct\s+on\s*\([^)]*\)|distinct|all|top\s+\d+)(?![\w$])/i.exec(text.slice(start)) + if (lead) start += lead[0].length + from = start + i = start - 1 + } + continue + } + if (ch === ",") { + items.push(stripped.slice(from, i).trim()) + from = i + 1 + } else if (boundary && word && /^(from|into|where|union|intersect|except|group|order|limit|qualify|having|window)$/.test(word)) { + items.push(stripped.slice(from, i).trim()) + return items.filter((one) => one !== "") + } + } + if (start >= 0) items.push(stripped.slice(from).trim()) + return items.filter((one) => one !== "") +} + +/** Words that end an expression (`x is null`, `case … end`) rather than alias it. */ +const ENDS_EXPRESSION = /^(?:null|true|false|end)$/i +/** Words after which the next word is an operand (`a and b`, `sum(x) over w`), not an alias. */ +const TAKES_OPERAND = /^(?:and|or|not|is|in|like|ilike|rlike|regexp|similar|to|between|case|when|then|else|over|collate|escape|interval|distinct|by|as|at|zone|any|all|some|exists|div|mod|xor)$/i + +/** + * A SELECT item's output name when the SQL gives one (`x as name`, `x name`, a bare `t.col`), `*` for a star, + * else null. + */ +export function itemName(item: string): string | null { + if (/^(?:[\w$]+\.)*\*(?:\s|$)/.test(item)) return "*" + const as = new RegExp(`\\s+as\\s+(${QUOTED}|[\\w$]+)$`, "i").exec(item) + if (as) return unquoteName(as[1]!) + const bare = new RegExp(`^(?:(?:${QUOTED}|[\\w$]+)\\.)*(${QUOTED}|[\\w$]+)$`).exec(item) + if (bare) return unquoteName(bare[1]!) + // An alias without AS: the last word, after an expression that ends in a name, a `)`, a quote or a literal. + const plain = new RegExp(`^([\\s\\S]*?[\\w$)"\`\\]'])\\s+(${QUOTED}|[\\w$]+)$`).exec(item) + if (!plain) return null + const before = /([\w$]+)\s*$/.exec(plain[1]!)?.[1] + if ((before && TAKES_OPERAND.test(before)) || /(? { + const out = new Set() + const call = new RegExp(COLUMNS_CALL.source, "gi") + // A quoted name is blanked whole, so a `)` or `columns(` inside one neither ends nor starts a call. + const text = stripped.replace(new RegExp(QUOTED, "g"), (q) => " ".repeat(q.length)) + for (let m = call.exec(text); m; m = call.exec(text)) { + let depth = 0 + let i = m.index + m[0].length - 1 + for (; i < text.length; i++) { + if (text[i] === "(") depth++ + else if (text[i] === ")" && --depth === 0) break + } + const alias = new RegExp(`^\\s*as\\s+(${QUOTED}|[\\w$]+)`, "i").exec(stripped.slice(i + 1)) + if (alias && /\\\d/.test(alias[1]!)) out.add(unquoteName(alias[1]!)) + } + return out +} + +/** + * The names DuckDB gives a SELECT list's repeated outputs, in list order: an output whose name is already used takes + * the first free `_N` (`val, val, val` is `val, val_1, val_2`; `id, id, amount as id_1` is `id, id_1, id_1_1`). + */ +function duckdbRepeats(outputs: string[]): string[] { + const used = new Set() + const out: string[] = [] + for (const raw of outputs) { + const name = raw.toLowerCase() + if (!used.has(name)) { + used.add(name) + continue + } + let k = 1 + while (used.has(`${name}_${k}`)) k++ + used.add(`${name}_${k}`) + out.push(`${name}_${k}`) + } + return out } /** @@ -504,14 +633,21 @@ export class ProjectLineage { if (n?.resource_type === "seed") for (const c of this.seedHeader(n)) add(c) if (this.traced(id) && !this.busy.has(id)) { // Names from the SQL that aren't the warehouse's: the engine's own name for an unaliased expression - // (`_1`, `_col_1`), and, where the SQL calls COLUMNS(), an alias template with a backslash and a digit - // (`as "col_\0"`). A documented name is kept as it is. + // (`_1`, `_col_1`), and the alias template a COLUMNS(...) item gives (`as "col_\0"`). A documented name is + // kept as it is. const sql = this.sqlOf(id) const stripped = sql ? stripSql(sql) : "" - const template = COLUMNS_CALL.test(stripped) - const real = (name: string) => !engineMade(name, stripped) && !(template && /\\\d/.test(name)) - for (const e of this.lineage(id).edges) if (real(e.to)) add(e.to) - if (sql) for (const c of this.outputColumns(sql)) if (real(c)) add(c) + const templates = columnsTemplates(stripped) + const edges = this.lineage(id).edges + // A name a parent column of the same name passes on (`select * from r` over r's real `_1`) is that column. + const passed = new Set(edges.filter((e) => e.fromCol === e.to).map((e) => e.to)) + const real = (name: string) => !templates.has(name.toLowerCase()) && (passed.has(name) || !engineMade(name, stripped)) + for (const e of edges) if (real(e.to)) add(e.to) + const outputs = sql ? this.outputColumns(sql).filter(real) : [] + for (const c of outputs) add(c) + // DuckDB keeps a repeated output under a new name (`val`, `val_1`): a `select *` over it reads both. Only + // the SELECT list decides the names; a documented `val_1` is that rename. + if (this.dialect === "duckdb") for (const c of duckdbRepeats(outputs)) add(c) } const out = [...found.values()] this.columnMemo.set(id, out) @@ -541,14 +677,24 @@ export class ProjectLineage { if (settled && this.traced(id)) { const sql = this.sqlOf(id) settled = !!sql && !/(? this.settledColumns(p) !== undefined) - : namesItsOutputs(this.outputColumns(sql), stripSql(sql))) + ? !this.lineage(id).unresolved && this.parents(id).every((p) => this.settledColumns(p) !== undefined) && + !this.renamesBesideStar(id, stripSql(sql)) + : namesItsOutputs(this.outputColumns(sql))) } const out = settled ? names : undefined this.settledMemo.set(id, out) return out } + /** Whether a bare `*` sits beside an output that repeats a name the star brings in (`select *, val as id`). */ + private renamesBesideStar(id: string, stripped: string): boolean { + const items = selectItems(stripped) + if (!items.some((item) => /^\*(?:\s|$)/.test(item))) return false + const brought = new Set(this.parents(id).flatMap((p) => [...(this.settledColumns(p) ?? [])])) + const explicit = items.map(itemName).filter((name): name is string => !!name && name !== "*") + return new Set(explicit).size !== explicit.length || explicit.some((name) => brought.has(name)) + } + lineage(id: string): ModelLineage { const memo = this.lineageMemo.get(id) if (memo) return memo @@ -628,10 +774,22 @@ export class ProjectLineage { } // The engine names an unaliased expression twice: `_N`, as if the value passed through unchanged, and // `_col_N`, how unknown. Keep the second; the warehouse picks the real name. + // A `_N` the SQL itself writes (`amount as _1`) is a real alias and stays. + const stripped = stripSql(raw) const named = new Set(edges.filter((e) => /^_col_\d+$/.test(e.to)).map((e) => `${e.from}|${e.fromCol}|${e.to.slice(5)}`)) for (let i = edges.length - 1; i >= 0; i--) { const e = edges[i]! - if (/^_\d+$/.test(e.to) && named.has(`${e.from}|${e.fromCol}|${e.to.slice(1)}`)) edges.splice(i, 1) + if (/^_\d+$/.test(e.to) && engineMade(e.to, stripped) && named.has(`${e.from}|${e.fromCol}|${e.to.slice(1)}`)) edges.splice(i, 1) + } + // Postgres and Redshift name an unaliased cast of a column after the column (`amount::numeric` is `amount`). + if (this.dialect === "postgres" || this.dialect === "redshift") { + const items = selectItems(stripped) + for (const e of edges) { + // A `_col_N` the SQL writes itself (`amount::numeric as _col_1`) is that output's name: keep it. + const n = engineMade(e.to, stripped) ? /^_col_(\d+)$/.exec(e.to)?.[1] : undefined + const name = n === undefined ? null : castColumn(items[Number(n)] ?? "") + if (name && !edges.some((o) => o.from === e.from && o.fromCol === e.fromCol && o.to === name)) e.to = name + } } const errors: unknown[] = Array.isArray(result.errors) ? result.errors : [] const firstError = (): string => { diff --git a/packages/opencode/test/altimate/column-answer.test.ts b/packages/opencode/test/altimate/column-answer.test.ts index 04628dcb35..c2bb551bdc 100644 --- a/packages/opencode/test/altimate/column-answer.test.ts +++ b/packages/opencode/test/altimate/column-answer.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path" import { ENGINE, MAX_ANSWER_COLUMNS, columnAnswer, describeAnswer, type ColumnAnswer } from "../../src/altimate/native/dbt/column-answer" import { columnReach, namesColumn } from "../../src/altimate/native/dbt/column-reach" import { dbtLineage } from "../../src/altimate/native/dbt/lineage" -import { clearLineageCaches, hasNestedWith, howOf, quotingIsClear, relationUse, shapeFlags, stripAggFilter, stripSql } from "../../src/altimate/native/dbt/project-lineage" +import { clearLineageCaches, hasNestedWith, howOf, itemName, quotingIsClear, relationUse, shapeFlags, stripAggFilter, stripSql } from "../../src/altimate/native/dbt/project-lineage" const dirs: string[] = [] afterEach(() => { @@ -843,7 +843,7 @@ describe("column answer: why a model may carry it", () => { const a = columnAnswer(write(base(`select id from ${T("f")} where id > 0`)), "f", "val") expect(reasons(a)).toEqual({ after: ["not_traced", null], r: ["not_traced", null] }) expect(a.counts).toMatchObject({ models: 0, may_carry_models: 2, not_traced_models: 2, model_level: 2 }) - expect(describeAnswer(a)).toContain("; 2 more of the 2 downstream may carry it, 2 of them with no sign (not traced)") + expect(describeAnswer(a)).toContain("; 2 more of the 2 downstream may carry it, 2 of them not traced (no sign either way)") }) test("a reader that names the column with no traced edge may carry it, and what sits past it is below it", () => { @@ -1186,3 +1186,180 @@ describe("both tools: names that aren't columns", () => { expect(JSON.stringify((lineage.raw_lineage as any).column_dict?.val ?? [])).toContain("__altimate_unknown_0__") }) }) + +// Names at the edges: aliases that look like the engine's own names, templates, casts, repeats. +describe("both tools: names at the edges", () => { + const nodes = (dialect: string, list: any[]) => ({ metadata: { adapter_type: dialect }, sources: {}, nodes: Object.fromEntries(list.map((n) => [n.unique_id, n])) }) + const carried = (a: ColumnAnswer) => a.columns.slice(1).map((c) => `${named(a, c.model)}.${c.column}`) + + test("a real _1 column a select * child inherits stays a column of the child", () => { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], `select id, amount as _1 from ${T("f")}`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + expect(() => columnReach(path, "child", "_1")).not.toThrow() + expect(carried(columnAnswer(path, "child", "_1", "upstream"))).toEqual(expect.arrayContaining(["r._1", "f.amount"])) + }) + + test("a real alias named like the engine's _N keeps its edge beside an engine-named twin", () => { + for (const sql of [`select amount as _1, amount as _col_1 from ${T("f")}`, `select id, amount * 2, amount as _1 from ${T("f")}`]) { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], sql), + model("child", ["r"], `select _1 as x from ${T("r")}`), + ])) + expect(carried(columnAnswer(path, "f", "amount"))).toEqual(expect.arrayContaining(["r._1", "child.x"])) + } + }) + + test("a repeated name beside a * doesn't settle: the warehouse renames it", () => { + // With AS, without AS, and after a quoted name that is a keyword. + for (const item of ["val as id", "val id", `val as "from", val as id`]) { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2 as val", ["id", "val"]), + model("r", ["f"], `select *, ${item} from ${T("f")}`), + model("child", ["r"], `select id_1 as x from ${T("r")}`), + ])) + expect(columnReach(path, "r", "id_1").models.map((one) => one.name)).toEqual(["child"]) + expect(() => columnAnswer(path, "r", "id_1")).not.toThrow() + } + }) + + test("an alias without AS is read, and the end of an expression isn't taken for one", () => { + const named = { "val id": "id", "count(*) n": "n", "t.col c2": "c2", "'' label": "label", '"My Col"': "my col", "a.b as c": "c", "f.*": "*" } + for (const [item, name] of Object.entries(named)) expect(itemName(item)).toBe(name) + for (const item of ["x is null", "x is not true", "case when a > 0 then 1 else 0 end", "sum(x) over w", "a and b", "a + b", + "a div b", "a mod b", "a xor b", "x between a and b", "interval '' day", "interval 1 day", "interval 1.5 hour", `x collate "C"`, "amount::numeric"]) + expect(itemName(item)).toBeNull() + }) + + test("a COLUMNS() alias without a \\N marker is the output's name, not a template", () => { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], `select id, columns('amount') as total, columns('amount') as "x_\\0" from ${T("f")}`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const lineage = dbtLineage({ manifest_path: path, model: "child" }) + const names = Object.keys((lineage.raw_lineage as any).column_dict ?? {}) + expect(names).toContain("total") + expect(names).not.toContain("x_\\0") + }) + + test("a quoted name holding a paren doesn't hide a COLUMNS() template", () => { + const path = write(nodes("duckdb", [ + model("f", [], `select 1 as id, 2.5 as "amount)"`, ["id", "amount)"]), + model("r", ["f"], `select id, columns("amount)") as "x_\\0" from ${T("f")}`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const lineage = dbtLineage({ manifest_path: path, model: "child" }) + expect(Object.keys((lineage.raw_lineage as any).column_dict ?? {})).not.toContain("x_\\0") + }) + + test("a COLUMNS() call elsewhere in the SQL leaves a real backslash alias a column", () => { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], `select id, amount as "col_\\0" from ${T("f")} where columns('amount') > 0`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const lineage = dbtLineage({ manifest_path: path, model: "child" }) + expect(Object.keys((lineage.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["col_\\0", "id"]) + }) + + test("on Postgres and Redshift a cast of a column is named after it, so the trace goes on", () => { + for (const dialect of ["postgres", "redshift"]) { + for (const cast of ["amount::numeric", "cast(amount as numeric)", "amount::int[]", "cast(amount as int[])", `amount::"numeric"`, "amount::numeric::int", + "amount::int[3]", "amount::pg_catalog.numeric", "amount::double precision", "amount::numeric(10, 2)"]) { + if (dialect === "redshift" && cast.includes("[")) continue // Redshift has no array types + const path = write(nodes(dialect, [ + model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), + model("r", ["orders"], `select order_id, ${cast} from ${T("orders")}`), + model("child", ["r"], `select order_id, amount from ${T("r")}`), + ])) + expect(carried(columnAnswer(path, "orders", "amount"))).toEqual(expect.arrayContaining(["r.amount", "child.amount"])) + const up = columnAnswer(path, "child", "amount", "upstream") + expect(up.may_carry.map((m) => m.name)).not.toContain("orders") + } + } + // A quoted name that is a keyword doesn't end the SELECT list, so the cast after it is still read. + const quoted = write(nodes("postgres", [ + model("orders", [], `select 1 as "order", 2.5 as amount`, ["order", "amount"]), + model("r", ["orders"], `select "order", amount::numeric from ${T("orders")}`), + ])) + expect(carried(columnAnswer(quoted, "orders", "amount"))).toEqual(["r.amount"]) + // `_col_N` counts SELECT items from 0, unnamed or not: each of two casts keeps its own column's name. + const two = write(nodes("postgres", [ + model("orders", [], "select 1 as order_id, 2 as status, 2.5 as amount", ["order_id", "status", "amount"]), + model("r", ["orders"], `select order_id::int, status, amount::numeric from ${T("orders")}`), + ])) + expect(carried(columnAnswer(two, "orders", "amount"))).toEqual(["r.amount"]) + expect(carried(columnAnswer(two, "orders", "order_id"))).toEqual(["r.order_id"]) + // Elsewhere the warehouse picks another name: the engine's label stays. + const duck = write(nodes("duckdb", [ + model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), + model("r", ["orders"], `select order_id, amount::decimal(10,2) from ${T("orders")}`), + ])) + expect(carried(columnAnswer(duck, "orders", "amount"))).toEqual(["r._col_1"]) + }) + + test("a column name that starts with all or distinct isn't read as a SELECT modifier", () => { + for (const name of ["allocation", "distinct_users", "all_time"]) { + const path = write(nodes("postgres", [ + model("f", [], `select 2.5 as ${name}`, [name]), + model("r", ["f"], `with q as (select * from ${T("f")}) select ${name}::numeric from q`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + expect(() => columnReach(path, "r", name)).not.toThrow() + expect(() => columnAnswer(path, "r", name)).not.toThrow() + const lineage = dbtLineage({ manifest_path: path, model: "child" }) + expect(Object.keys((lineage.raw_lineage as any).column_dict ?? {})).toContain(name) + } + }) + + test("an expression that only starts with a cast isn't named after the column, and a _col_N the SQL writes stays", () => { + for (const item of ["amount::numeric is not null", "amount::int in (1, 2)", "cast(amount as int) between 1 and 2", "amount::numeric as _col_1"]) { + const path = write(nodes("postgres", [ + model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), + model("r", ["orders"], `select order_id, ${item} from ${T("orders")}`), + ])) + expect(carried(columnAnswer(path, "orders", "amount"))).toEqual(["r._col_1"]) + } + }) + + test("DuckDB names a repeated output from a join val_1, and a select * child lists it", () => { + const path = write(nodes("duckdb", [ + model("a", [], "select 1 as id, 2 as val", ["id", "val"]), + model("b", [], "select 1 as id, 3 as val", ["id", "val"]), + model("r", ["a", "b"], `select a.id, a.val, b.val from ${T("a")} a join ${T("b")} b on a.id = b.id`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const lineage = dbtLineage({ manifest_path: path, model: "child" }) + expect(Object.keys((lineage.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "val", "val_1"]) + // Docs that list the rename add no `val_2`. + const documented = write(nodes("duckdb", [ + model("a", [], "select 1 as id, 2 as val", ["id", "val"]), + model("b", [], "select 1 as id, 3 as val", ["id", "val"]), + model("r", ["a", "b"], `select a.id, a.val, b.val from ${T("a")} a join ${T("b")} b on a.id = b.id`, ["id", "val", "val_1"]), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const listed = dbtLineage({ manifest_path: documented, model: "child" }) + expect(Object.keys((listed.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "val", "val_1"]) + // In list order, as DuckDB names them: `id, id, amount as id_1` is `id, id_1, id_1_1`, with no `id_2`. + const clash = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], `select id, id, amount as id_1 from ${T("f")}`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const named = dbtLineage({ manifest_path: clash, model: "child" }) + expect(Object.keys((named.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "id_1", "id_1_1"]) + }) + + test("the text line says how many may-carry models are only not traced, without suggesting no effect", () => { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2 as val", ["id", "val"]), + model("r", ["f"], `select id from ${T("f")} where id > 0`), + ])) + expect(describeAnswer(columnAnswer(path, "f", "val"))).toContain("; 1 more of the 1 downstream may carry it, 1 of them not traced (no sign either way)") + }) +}) +