From 755204dac6f14a4b9dea4db3359d804f5fe2aebf Mon Sep 17 00:00:00 2001 From: "Ralph Sto. Domingo" Date: Fri, 13 Nov 2026 20:49:20 +0800 Subject: [PATCH 1/5] fix: column names at the edges of the answer A `_N` a parent passes on or the SQL writes stays a column; only a COLUMNS() item's own alias is a template; a name repeated beside `*` doesn't settle; Postgres and Redshift name a cast after its column; DuckDB's repeated outputs get `_1` names. The text line reads "not traced (no sign either way)". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VEmKQwEiHMwQ7BdwgPo9sb --- .../src/altimate/native/dbt/column-answer.ts | 2 +- .../altimate/native/dbt/project-lineage.ts | 147 ++++++++++++++++-- .../test/altimate/column-answer.test.ts | 90 ++++++++++- 3 files changed, 225 insertions(+), 14 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/column-answer.ts b/packages/opencode/src/altimate/native/dbt/column-answer.ts index 525ad3d9eb..24fc6b1ebd 100644 --- a/packages/opencode/src/altimate/native/dbt/column-answer.ts +++ b/packages/opencode/src/altimate/native/dbt/column-answer.ts @@ -465,6 +465,6 @@ export function describeAnswer(a: ColumnAnswer): string { const c = a.counts const what = a.direction === "downstream" ? "Column impact" : "Column trace (upstream)" const reach = `${c.columns} column${c.columns === 1 ? "" : "s"} in ${c.models} model${c.models === 1 ? "" : "s"}, ${c.hops} hop${c.hops === 1 ? "" : "s"}, ${c.ends} end${c.ends === 1 ? "" : "s"}` - const tiers = `${c.may_carry_models} more of the ${c.model_level} ${a.direction} may carry it${c.not_traced_models ? `, ${c.not_traced_models} of them with no sign (not traced)` : ""}` + const tiers = `${c.may_carry_models} more of the ${c.model_level} ${a.direction} may carry it${c.not_traced_models ? `, ${c.not_traced_models} of them not traced (no sign either way)` : ""}` return `${what}: ${reach}; ${tiers}${a.truncated ? " (truncated)" : ""} (metadata.column_answer)` } diff --git a/packages/opencode/src/altimate/native/dbt/project-lineage.ts b/packages/opencode/src/altimate/native/dbt/project-lineage.ts index ba659ff50b..e51a895704 100644 --- a/packages/opencode/src/altimate/native/dbt/project-lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/project-lineage.ts @@ -102,10 +102,106 @@ function engineMade(name: string, stripped: string): boolean { return ENGINE_NAME.test(name) && !new RegExp(`(? name.toLowerCase()) - return names.length > 0 && new Set(names).size === names.length && !names.some((name) => engineMade(name, stripped)) + return names.length > 0 && new Set(names).size === names.length +} + +const QUOTED = '"(?:[^"]|"")*"|`[^`]*`|\\[[^\\]]*\\]' +const unquoteName = (name: string) => name.replace(/^["`[]|["`\]]$/g, "").toLowerCase() + +/** + * The items of the SQL's outermost SELECT list (the first one outside parentheses, so after any WITH), split at + * top-level commas, from literal- and comment-stripped SQL; [] when there is none. + */ +export function selectItems(stripped: string): string[] { + const text = stripped.replace(new RegExp(QUOTED, "g"), (q) => q.replace(/[(),]/g, " ")) + let depth = 0 + let start = -1 + const items: string[] = [] + let from = 0 + for (let i = 0; i < text.length; i++) { + const ch = text[i]! + if (ch === "(") depth++ + else if (ch === ")") depth-- + if (depth !== 0) continue + const word = /^[a-z_]+/i.exec(text.slice(i))?.[0]?.toLowerCase() + const boundary = i === 0 || !/[\w$]/.test(text[i - 1]!) + if (start < 0) { + if (boundary && word === "select") { + start = i + 6 + const lead = /^\s+(?:distinct\s+on\s*\([^)]*\)|distinct|all|top\s+\d+)/i.exec(text.slice(start)) + if (lead) start += lead[0].length + from = start + i = start - 1 + } + continue + } + if (ch === ",") { + items.push(stripped.slice(from, i).trim()) + from = i + 1 + } else if (boundary && word && /^(from|into|where|union|intersect|except|group|order|limit|qualify|having|window)$/.test(word)) { + items.push(stripped.slice(from, i).trim()) + return items.filter((one) => one !== "") + } + } + if (start >= 0) items.push(stripped.slice(from).trim()) + return items.filter((one) => one !== "") +} + +/** A SELECT item's output name when the SQL gives one (`x as name`, a bare `t.col`), `*` for a star, else null. */ +export function itemName(item: string): string | null { + if (/^(?:[\w$]+\.)*\*(?:\s|$)/.test(item)) return "*" + const as = new RegExp(`\\s+as\\s+(${QUOTED}|[\\w$]+)$`, "i").exec(item) + if (as) return unquoteName(as[1]!) + const bare = new RegExp(`^(?:(?:${QUOTED}|[\\w$]+)\\.)*(${QUOTED}|[\\w$]+)$`).exec(item) + return bare ? unquoteName(bare[1]!) : null +} + +/** The column a SELECT item casts, unaliased (`amount::numeric`, `cast(t.amount as numeric)`); null otherwise. */ +function castColumn(item: string): string | null { + const ref = `(?:(?:${QUOTED}|[\\w$]+)\\.)*(${QUOTED}|[\\w$]+)` + const m = new RegExp(`^${ref}\\s*::\\s*[\\w\\s(),]+$`, "i").exec(item) ?? new RegExp(`^cast\\s*\\(\\s*${ref}\\s+as\\s+[\\w\\s(),]+\\)$`, "i").exec(item) + return m ? unquoteName(m[1]!) : null +} + +/** The names `COLUMNS(...) AS "…"` items give (DuckDB alias templates such as `"col_\0"`), lower-cased. */ +function columnsTemplates(stripped: string): Set { + const out = new Set() + const call = new RegExp(COLUMNS_CALL.source, "gi") + for (let m = call.exec(stripped); m; m = call.exec(stripped)) { + let depth = 0 + let i = m.index + m[0].length - 1 + for (; i < stripped.length; i++) { + if (stripped[i] === "(") depth++ + else if (stripped[i] === ")" && --depth === 0) break + } + const alias = new RegExp(`^\\s*as\\s+(${QUOTED}|[\\w$]+)`, "i").exec(stripped.slice(i + 1)) + if (alias) out.add(unquoteName(alias[1]!)) + } + return out +} + +/** DuckDB's names for a SELECT list's repeated outputs: the second `val` is `val_1`, the third `val_2`. */ +function duckdbRepeats(outputs: string[], taken: Set): string[] { + const seen = new Map() + const out: string[] = [] + for (const raw of outputs) { + const name = raw.toLowerCase() + const n = seen.get(name) ?? 0 + seen.set(name, n + 1) + if (n === 0) continue + let k = n + while (taken.has(`${name}_${k}`)) k++ + taken.add(`${name}_${k}`) + out.push(`${name}_${k}`) + } + return out } /** @@ -504,14 +600,20 @@ export class ProjectLineage { if (n?.resource_type === "seed") for (const c of this.seedHeader(n)) add(c) if (this.traced(id) && !this.busy.has(id)) { // Names from the SQL that aren't the warehouse's: the engine's own name for an unaliased expression - // (`_1`, `_col_1`), and, where the SQL calls COLUMNS(), an alias template with a backslash and a digit - // (`as "col_\0"`). A documented name is kept as it is. + // (`_1`, `_col_1`), and the alias template a COLUMNS(...) item gives (`as "col_\0"`). A documented name is + // kept as it is. const sql = this.sqlOf(id) const stripped = sql ? stripSql(sql) : "" - const template = COLUMNS_CALL.test(stripped) - const real = (name: string) => !engineMade(name, stripped) && !(template && /\\\d/.test(name)) - for (const e of this.lineage(id).edges) if (real(e.to)) add(e.to) - if (sql) for (const c of this.outputColumns(sql)) if (real(c)) add(c) + const templates = columnsTemplates(stripped) + const edges = this.lineage(id).edges + // A name a parent column of the same name passes on (`select * from r` over r's real `_1`) is that column. + const passed = new Set(edges.filter((e) => e.fromCol === e.to).map((e) => e.to)) + const real = (name: string) => !templates.has(name.toLowerCase()) && (passed.has(name) || !engineMade(name, stripped)) + for (const e of edges) if (real(e.to)) add(e.to) + const outputs = sql ? this.outputColumns(sql).filter(real) : [] + for (const c of outputs) add(c) + // DuckDB keeps a repeated output under a new name (`val`, `val_1`): a `select *` over it reads both. + if (this.dialect === "duckdb") for (const c of duckdbRepeats(outputs, new Set(found.keys()))) add(c) } const out = [...found.values()] this.columnMemo.set(id, out) @@ -541,14 +643,24 @@ export class ProjectLineage { if (settled && this.traced(id)) { const sql = this.sqlOf(id) settled = !!sql && !/(? this.settledColumns(p) !== undefined) - : namesItsOutputs(this.outputColumns(sql), stripSql(sql))) + ? !this.lineage(id).unresolved && this.parents(id).every((p) => this.settledColumns(p) !== undefined) && + !this.renamesBesideStar(id, stripSql(sql)) + : namesItsOutputs(this.outputColumns(sql))) } const out = settled ? names : undefined this.settledMemo.set(id, out) return out } + /** Whether a bare `*` sits beside an output that repeats a name the star brings in (`select *, val as id`). */ + private renamesBesideStar(id: string, stripped: string): boolean { + const items = selectItems(stripped) + if (!items.some((item) => /^\*(?:\s|$)/.test(item))) return false + const brought = new Set(this.parents(id).flatMap((p) => [...(this.settledColumns(p) ?? [])])) + const explicit = items.map(itemName).filter((name): name is string => !!name && name !== "*") + return new Set(explicit).size !== explicit.length || explicit.some((name) => brought.has(name)) + } + lineage(id: string): ModelLineage { const memo = this.lineageMemo.get(id) if (memo) return memo @@ -628,10 +740,21 @@ export class ProjectLineage { } // The engine names an unaliased expression twice: `_N`, as if the value passed through unchanged, and // `_col_N`, how unknown. Keep the second; the warehouse picks the real name. + // A `_N` the SQL itself writes (`amount as _1`) is a real alias and stays. + const stripped = stripSql(raw) const named = new Set(edges.filter((e) => /^_col_\d+$/.test(e.to)).map((e) => `${e.from}|${e.fromCol}|${e.to.slice(5)}`)) for (let i = edges.length - 1; i >= 0; i--) { const e = edges[i]! - if (/^_\d+$/.test(e.to) && named.has(`${e.from}|${e.fromCol}|${e.to.slice(1)}`)) edges.splice(i, 1) + if (/^_\d+$/.test(e.to) && engineMade(e.to, stripped) && named.has(`${e.from}|${e.fromCol}|${e.to.slice(1)}`)) edges.splice(i, 1) + } + // Postgres and Redshift name an unaliased cast of a column after the column (`amount::numeric` is `amount`). + if (this.dialect === "postgres" || this.dialect === "redshift") { + const items = selectItems(stripped) + for (const e of edges) { + const n = /^_col_(\d+)$/.exec(e.to)?.[1] + const name = n === undefined ? null : castColumn(items[Number(n)] ?? "") + if (name && !edges.some((o) => o.from === e.from && o.fromCol === e.fromCol && o.to === name)) e.to = name + } } const errors: unknown[] = Array.isArray(result.errors) ? result.errors : [] const firstError = (): string => { diff --git a/packages/opencode/test/altimate/column-answer.test.ts b/packages/opencode/test/altimate/column-answer.test.ts index 04628dcb35..91e5144a8a 100644 --- a/packages/opencode/test/altimate/column-answer.test.ts +++ b/packages/opencode/test/altimate/column-answer.test.ts @@ -843,7 +843,7 @@ describe("column answer: why a model may carry it", () => { const a = columnAnswer(write(base(`select id from ${T("f")} where id > 0`)), "f", "val") expect(reasons(a)).toEqual({ after: ["not_traced", null], r: ["not_traced", null] }) expect(a.counts).toMatchObject({ models: 0, may_carry_models: 2, not_traced_models: 2, model_level: 2 }) - expect(describeAnswer(a)).toContain("; 2 more of the 2 downstream may carry it, 2 of them with no sign (not traced)") + expect(describeAnswer(a)).toContain("; 2 more of the 2 downstream may carry it, 2 of them not traced (no sign either way)") }) test("a reader that names the column with no traced edge may carry it, and what sits past it is below it", () => { @@ -1186,3 +1186,91 @@ describe("both tools: names that aren't columns", () => { expect(JSON.stringify((lineage.raw_lineage as any).column_dict?.val ?? [])).toContain("__altimate_unknown_0__") }) }) + +// Names at the edges: aliases that look like the engine's own names, templates, casts, repeats. +describe("both tools: names at the edges", () => { + const nodes = (dialect: string, list: any[]) => ({ metadata: { adapter_type: dialect }, sources: {}, nodes: Object.fromEntries(list.map((n) => [n.unique_id, n])) }) + const carried = (a: ColumnAnswer) => a.columns.slice(1).map((c) => `${named(a, c.model)}.${c.column}`) + + test("a real _1 column a select * child inherits stays a column of the child", () => { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], `select id, amount as _1 from ${T("f")}`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + expect(() => columnReach(path, "child", "_1")).not.toThrow() + expect(carried(columnAnswer(path, "child", "_1", "upstream"))).toEqual(expect.arrayContaining(["r._1", "f.amount"])) + }) + + test("a real alias named like the engine's _N keeps its edge beside an engine-named twin", () => { + for (const sql of [`select amount as _1, amount as _col_1 from ${T("f")}`, `select id, amount * 2, amount as _1 from ${T("f")}`]) { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], sql), + model("child", ["r"], `select _1 as x from ${T("r")}`), + ])) + expect(carried(columnAnswer(path, "f", "amount"))).toEqual(expect.arrayContaining(["r._1", "child.x"])) + } + }) + + test("a repeated name beside a * doesn't settle: the warehouse renames it", () => { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2 as val", ["id", "val"]), + model("r", ["f"], `select *, val as id from ${T("f")}`), + model("child", ["r"], `select id_1 as x from ${T("r")}`), + ])) + expect(columnReach(path, "r", "id_1").models.map((one) => one.name)).toEqual(["child"]) + expect(() => columnAnswer(path, "r", "id_1")).not.toThrow() + }) + + test("a COLUMNS() call elsewhere in the SQL leaves a real backslash alias a column", () => { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], `select id, amount as "col_\\0" from ${T("f")} where columns('amount') > 0`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const lineage = dbtLineage({ manifest_path: path, model: "child" }) + expect(Object.keys((lineage.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["col_\\0", "id"]) + }) + + test("on Postgres and Redshift a cast of a column is named after it, so the trace goes on", () => { + for (const dialect of ["postgres", "redshift"]) { + for (const cast of ["amount::numeric", "cast(amount as numeric)"]) { + const path = write(nodes(dialect, [ + model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), + model("r", ["orders"], `select order_id, ${cast} from ${T("orders")}`), + model("child", ["r"], `select order_id, amount from ${T("r")}`), + ])) + expect(carried(columnAnswer(path, "orders", "amount"))).toEqual(expect.arrayContaining(["r.amount", "child.amount"])) + const up = columnAnswer(path, "child", "amount", "upstream") + expect(up.may_carry.map((m) => m.name)).not.toContain("orders") + } + } + // Elsewhere the warehouse picks another name: the engine's label stays. + const duck = write(nodes("duckdb", [ + model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), + model("r", ["orders"], `select order_id, amount::decimal(10,2) from ${T("orders")}`), + ])) + expect(carried(columnAnswer(duck, "orders", "amount"))).toEqual(["r._col_1"]) + }) + + test("DuckDB names a repeated output from a join val_1, and a select * child lists it", () => { + const path = write(nodes("duckdb", [ + model("a", [], "select 1 as id, 2 as val", ["id", "val"]), + model("b", [], "select 1 as id, 3 as val", ["id", "val"]), + model("r", ["a", "b"], `select a.id, a.val, b.val from ${T("a")} a join ${T("b")} b on a.id = b.id`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const lineage = dbtLineage({ manifest_path: path, model: "child" }) + expect(Object.keys((lineage.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "val", "val_1"]) + }) + + test("the text line says how many may-carry models are only not traced, without suggesting no effect", () => { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2 as val", ["id", "val"]), + model("r", ["f"], `select id from ${T("f")} where id > 0`), + ])) + expect(describeAnswer(columnAnswer(path, "f", "val"))).toContain("; 1 more of the 1 downstream may carry it, 1 of them not traced (no sign either way)") + }) +}) + From 7e7c65786b82671d90c3cc399432a7783b39333e Mon Sep 17 00:00:00 2001 From: "Ralph Sto. Domingo" Date: Mon, 16 Nov 2026 14:47:51 +0800 Subject: [PATCH 2/5] fix: read SELECT items whole at the column answer's edges - A quoted name ("from", "a)") no longer ends the SELECT list or a COLUMNS() call - An alias without AS (`val id`) counts beside a `*` - Only a COLUMNS() alias holding a `\N` marker is a template; `as total` is a column Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VEmKQwEiHMwQ7BdwgPo9sb --- .../altimate/native/dbt/project-lineage.ts | 38 +++++++++++++---- .../test/altimate/column-answer.test.ts | 41 ++++++++++++++++--- 2 files changed, 65 insertions(+), 14 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/project-lineage.ts b/packages/opencode/src/altimate/native/dbt/project-lineage.ts index e51a895704..45579833e5 100644 --- a/packages/opencode/src/altimate/native/dbt/project-lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/project-lineage.ts @@ -120,7 +120,8 @@ const unquoteName = (name: string) => name.replace(/^["`[]|["`\]]$/g, "").toLowe * top-level commas, from literal- and comment-stripped SQL; [] when there is none. */ export function selectItems(stripped: string): string[] { - const text = stripped.replace(new RegExp(QUOTED, "g"), (q) => q.replace(/[(),]/g, " ")) + // A quoted name is blanked whole, so `"from"` or `"a,b"` ends no item. + const text = stripped.replace(new RegExp(QUOTED, "g"), (q) => " ".repeat(q.length)) let depth = 0 let start = -1 const items: string[] = [] @@ -154,13 +155,27 @@ export function selectItems(stripped: string): string[] { return items.filter((one) => one !== "") } -/** A SELECT item's output name when the SQL gives one (`x as name`, a bare `t.col`), `*` for a star, else null. */ +/** Words that end an expression (`x is null`, `case … end`) rather than alias it. */ +const ENDS_EXPRESSION = /^(?:null|true|false|end)$/i +/** Words after which the next word is an operand (`a and b`, `sum(x) over w`), not an alias. */ +const TAKES_OPERAND = /^(?:and|or|not|is|in|like|ilike|rlike|regexp|similar|to|between|case|when|then|else|over|collate|escape|interval|distinct|by|as|at|zone|any|all|some|exists)$/i + +/** + * A SELECT item's output name when the SQL gives one (`x as name`, `x name`, a bare `t.col`), `*` for a star, + * else null. + */ export function itemName(item: string): string | null { if (/^(?:[\w$]+\.)*\*(?:\s|$)/.test(item)) return "*" const as = new RegExp(`\\s+as\\s+(${QUOTED}|[\\w$]+)$`, "i").exec(item) if (as) return unquoteName(as[1]!) const bare = new RegExp(`^(?:(?:${QUOTED}|[\\w$]+)\\.)*(${QUOTED}|[\\w$]+)$`).exec(item) - return bare ? unquoteName(bare[1]!) : null + if (bare) return unquoteName(bare[1]!) + // An alias without AS: the last word, after an expression that ends in a name, a `)`, a quote or a literal. + const plain = new RegExp(`^([\\s\\S]*?[\\w$)"\`\\]'])\\s+(${QUOTED}|[\\w$]+)$`).exec(item) + if (!plain) return null + const before = /([\w$]+)\s*$/.exec(plain[1]!)?.[1] + if ((before && TAKES_OPERAND.test(before)) || /(? { const out = new Set() const call = new RegExp(COLUMNS_CALL.source, "gi") - for (let m = call.exec(stripped); m; m = call.exec(stripped)) { + // A quoted name is blanked whole, so a `)` or `columns(` inside one neither ends nor starts a call. + const text = stripped.replace(new RegExp(QUOTED, "g"), (q) => " ".repeat(q.length)) + for (let m = call.exec(text); m; m = call.exec(text)) { let depth = 0 let i = m.index + m[0].length - 1 - for (; i < stripped.length; i++) { - if (stripped[i] === "(") depth++ - else if (stripped[i] === ")" && --depth === 0) break + for (; i < text.length; i++) { + if (text[i] === "(") depth++ + else if (text[i] === ")" && --depth === 0) break } const alias = new RegExp(`^\\s*as\\s+(${QUOTED}|[\\w$]+)`, "i").exec(stripped.slice(i + 1)) - if (alias) out.add(unquoteName(alias[1]!)) + if (alias && /\\\d/.test(alias[1]!)) out.add(unquoteName(alias[1]!)) } return out } diff --git a/packages/opencode/test/altimate/column-answer.test.ts b/packages/opencode/test/altimate/column-answer.test.ts index 91e5144a8a..1dcb5f482e 100644 --- a/packages/opencode/test/altimate/column-answer.test.ts +++ b/packages/opencode/test/altimate/column-answer.test.ts @@ -1214,13 +1214,38 @@ describe("both tools: names at the edges", () => { }) test("a repeated name beside a * doesn't settle: the warehouse renames it", () => { + // With AS, without AS, and after a quoted name that is a keyword. + for (const item of ["val as id", "val id", `val as "from", val as id`]) { + const path = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2 as val", ["id", "val"]), + model("r", ["f"], `select *, ${item} from ${T("f")}`), + model("child", ["r"], `select id_1 as x from ${T("r")}`), + ])) + expect(columnReach(path, "r", "id_1").models.map((one) => one.name)).toEqual(["child"]) + expect(() => columnAnswer(path, "r", "id_1")).not.toThrow() + } + }) + + test("a COLUMNS() alias without a \\N marker is the output's name, not a template", () => { const path = write(nodes("duckdb", [ - model("f", [], "select 1 as id, 2 as val", ["id", "val"]), - model("r", ["f"], `select *, val as id from ${T("f")}`), - model("child", ["r"], `select id_1 as x from ${T("r")}`), + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], `select id, columns('amount') as total, columns('amount') as "x_\\0" from ${T("f")}`), + model("child", ["r"], `select * from ${T("r")}`), ])) - expect(columnReach(path, "r", "id_1").models.map((one) => one.name)).toEqual(["child"]) - expect(() => columnAnswer(path, "r", "id_1")).not.toThrow() + const lineage = dbtLineage({ manifest_path: path, model: "child" }) + const names = Object.keys((lineage.raw_lineage as any).column_dict ?? {}) + expect(names).toContain("total") + expect(names).not.toContain("x_\\0") + }) + + test("a quoted name holding a paren doesn't hide a COLUMNS() template", () => { + const path = write(nodes("duckdb", [ + model("f", [], `select 1 as id, 2.5 as "amount)"`, ["id", "amount)"]), + model("r", ["f"], `select id, columns("amount)") as "x_\\0" from ${T("f")}`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const lineage = dbtLineage({ manifest_path: path, model: "child" }) + expect(Object.keys((lineage.raw_lineage as any).column_dict ?? {})).not.toContain("x_\\0") }) test("a COLUMNS() call elsewhere in the SQL leaves a real backslash alias a column", () => { @@ -1246,6 +1271,12 @@ describe("both tools: names at the edges", () => { expect(up.may_carry.map((m) => m.name)).not.toContain("orders") } } + // A quoted name that is a keyword doesn't end the SELECT list, so the cast after it is still read. + const quoted = write(nodes("postgres", [ + model("orders", [], `select 1 as "order", 2.5 as amount`, ["order", "amount"]), + model("r", ["orders"], `select "order", amount::numeric from ${T("orders")}`), + ])) + expect(carried(columnAnswer(quoted, "orders", "amount"))).toEqual(["r.amount"]) // Elsewhere the warehouse picks another name: the engine's label stays. const duck = write(nodes("duckdb", [ model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), From fc5166ba6062e95b486c48ca4446ee540f27e588 Mon Sep 17 00:00:00 2001 From: "Ralph Sto. Domingo" Date: Mon, 16 Nov 2026 15:31:55 +0800 Subject: [PATCH 3/5] fix: name array and quoted casts, and add no extra DuckDB repeat - Postgres/Redshift: `amount::int[]`, `amount::"numeric"`, `amount::numeric::int` are named `amount` - DuckDB: a documented `val_1` adds no `val_2` - `a div b` and `interval 1 day` aren't aliases - A test pins `_col_N` to the SELECT position Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VEmKQwEiHMwQ7BdwgPo9sb --- .../altimate/native/dbt/project-lineage.ts | 14 ++++++---- .../test/altimate/column-answer.test.ts | 28 +++++++++++++++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/project-lineage.ts b/packages/opencode/src/altimate/native/dbt/project-lineage.ts index 45579833e5..49f7e321da 100644 --- a/packages/opencode/src/altimate/native/dbt/project-lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/project-lineage.ts @@ -158,7 +158,7 @@ export function selectItems(stripped: string): string[] { /** Words that end an expression (`x is null`, `case … end`) rather than alias it. */ const ENDS_EXPRESSION = /^(?:null|true|false|end)$/i /** Words after which the next word is an operand (`a and b`, `sum(x) over w`), not an alias. */ -const TAKES_OPERAND = /^(?:and|or|not|is|in|like|ilike|rlike|regexp|similar|to|between|case|when|then|else|over|collate|escape|interval|distinct|by|as|at|zone|any|all|some|exists)$/i +const TAKES_OPERAND = /^(?:and|or|not|is|in|like|ilike|rlike|regexp|similar|to|between|case|when|then|else|over|collate|escape|interval|distinct|by|as|at|zone|any|all|some|exists|div|mod|xor)$/i /** * A SELECT item's output name when the SQL gives one (`x as name`, `x name`, a bare `t.col`), `*` for a star, @@ -174,14 +174,16 @@ export function itemName(item: string): string | null { const plain = new RegExp(`^([\\s\\S]*?[\\w$)"\`\\]'])\\s+(${QUOTED}|[\\w$]+)$`).exec(item) if (!plain) return null const before = /([\w$]+)\s*$/.exec(plain[1]!)?.[1] - if ((before && TAKES_OPERAND.test(before)) || /(? e.to), ...outputs].filter(real).map((name) => name.toLowerCase())) + if (this.dialect === "duckdb") for (const c of duckdbRepeats(outputs, made)) add(c) } const out = [...found.values()] this.columnMemo.set(id, out) diff --git a/packages/opencode/test/altimate/column-answer.test.ts b/packages/opencode/test/altimate/column-answer.test.ts index 1dcb5f482e..8112e1d8a2 100644 --- a/packages/opencode/test/altimate/column-answer.test.ts +++ b/packages/opencode/test/altimate/column-answer.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path" import { ENGINE, MAX_ANSWER_COLUMNS, columnAnswer, describeAnswer, type ColumnAnswer } from "../../src/altimate/native/dbt/column-answer" import { columnReach, namesColumn } from "../../src/altimate/native/dbt/column-reach" import { dbtLineage } from "../../src/altimate/native/dbt/lineage" -import { clearLineageCaches, hasNestedWith, howOf, quotingIsClear, relationUse, shapeFlags, stripAggFilter, stripSql } from "../../src/altimate/native/dbt/project-lineage" +import { clearLineageCaches, hasNestedWith, howOf, itemName, quotingIsClear, relationUse, shapeFlags, stripAggFilter, stripSql } from "../../src/altimate/native/dbt/project-lineage" const dirs: string[] = [] afterEach(() => { @@ -1226,6 +1226,14 @@ describe("both tools: names at the edges", () => { } }) + test("an alias without AS is read, and the end of an expression isn't taken for one", () => { + const named = { "val id": "id", "count(*) n": "n", "t.col c2": "c2", "'' label": "label", '"My Col"': "my col", "a.b as c": "c", "f.*": "*" } + for (const [item, name] of Object.entries(named)) expect(itemName(item)).toBe(name) + for (const item of ["x is null", "x is not true", "case when a > 0 then 1 else 0 end", "sum(x) over w", "a and b", "a + b", + "a div b", "a mod b", "a xor b", "x between a and b", "interval '' day", "interval 1 day", "interval 1.5 hour", `x collate "C"`, "amount::numeric"]) + expect(itemName(item)).toBeNull() + }) + test("a COLUMNS() alias without a \\N marker is the output's name, not a template", () => { const path = write(nodes("duckdb", [ model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), @@ -1260,7 +1268,7 @@ describe("both tools: names at the edges", () => { test("on Postgres and Redshift a cast of a column is named after it, so the trace goes on", () => { for (const dialect of ["postgres", "redshift"]) { - for (const cast of ["amount::numeric", "cast(amount as numeric)"]) { + for (const cast of ["amount::numeric", "cast(amount as numeric)", "amount::int[]", "cast(amount as int[])", `amount::"numeric"`, "amount::numeric::int"]) { const path = write(nodes(dialect, [ model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), model("r", ["orders"], `select order_id, ${cast} from ${T("orders")}`), @@ -1277,6 +1285,13 @@ describe("both tools: names at the edges", () => { model("r", ["orders"], `select "order", amount::numeric from ${T("orders")}`), ])) expect(carried(columnAnswer(quoted, "orders", "amount"))).toEqual(["r.amount"]) + // `_col_N` counts SELECT items from 0, unnamed or not: each of two casts keeps its own column's name. + const two = write(nodes("postgres", [ + model("orders", [], "select 1 as order_id, 2 as status, 2.5 as amount", ["order_id", "status", "amount"]), + model("r", ["orders"], `select order_id::int, status, amount::numeric from ${T("orders")}`), + ])) + expect(carried(columnAnswer(two, "orders", "amount"))).toEqual(["r.amount"]) + expect(carried(columnAnswer(two, "orders", "order_id"))).toEqual(["r.order_id"]) // Elsewhere the warehouse picks another name: the engine's label stays. const duck = write(nodes("duckdb", [ model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), @@ -1294,6 +1309,15 @@ describe("both tools: names at the edges", () => { ])) const lineage = dbtLineage({ manifest_path: path, model: "child" }) expect(Object.keys((lineage.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "val", "val_1"]) + // Docs that list the rename add no `val_2`. + const documented = write(nodes("duckdb", [ + model("a", [], "select 1 as id, 2 as val", ["id", "val"]), + model("b", [], "select 1 as id, 3 as val", ["id", "val"]), + model("r", ["a", "b"], `select a.id, a.val, b.val from ${T("a")} a join ${T("b")} b on a.id = b.id`, ["id", "val", "val_1"]), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const listed = dbtLineage({ manifest_path: documented, model: "child" }) + expect(Object.keys((listed.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "val", "val_1"]) }) test("the text line says how many may-carry models are only not traced, without suggesting no effect", () => { From f49e1b7a71ee1ad300ab27b9aa4ad393d4510f04 Mon Sep 17 00:00:00 2001 From: "Ralph Sto. Domingo" Date: Mon, 16 Nov 2026 16:48:17 +0800 Subject: [PATCH 4/5] fix: only a whole cast of a column is named after it - Postgres/Redshift: `amount::numeric is not null` or `cast(amount as int) between 1 and 2` keeps `_col_1`, as on main - A `_col_N` the SQL writes (`as _col_1`) is never renamed - Sized arrays and qualified types (`int[3]`, `pg_catalog.numeric`) are casts Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VEmKQwEiHMwQ7BdwgPo9sb --- .../src/altimate/native/dbt/project-lineage.ts | 15 +++++++++++---- .../opencode/test/altimate/column-answer.test.ts | 14 +++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/project-lineage.ts b/packages/opencode/src/altimate/native/dbt/project-lineage.ts index 49f7e321da..f6d15a72c3 100644 --- a/packages/opencode/src/altimate/native/dbt/project-lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/project-lineage.ts @@ -181,12 +181,18 @@ export function itemName(item: string): string | null { /** The column a SELECT item casts, unaliased (`amount::numeric`, `cast(t.amount as numeric)`); null otherwise. */ function castColumn(item: string): string | null { const ref = `(?:(?:${QUOTED}|[\\w$]+)\\.)*(${QUOTED}|[\\w$]+)` - // A type name: words, a size (`numeric(10, 2)`), array marks (`int[]`), a quoted name, a further `::` cast. - const type = `(?:[\\w\\s(),:]|\\[\\]|"(?:[^"]|"")*")+` + // A type name: words (`double precision`, `pg_catalog.int4`), a size (`numeric(10, 2)`), array marks (`int[]`, + // `int[3]`), a quoted name, a further `::` cast. + const type = `((?:[\\w\\s(),:.]|\\[\\d*\\]|"(?:[^"]|"")*")+)` const m = new RegExp(`^${ref}\\s*::\\s*${type}$`, "i").exec(item) ?? new RegExp(`^cast\\s*\\(\\s*${ref}\\s+as\\s+${type}\\)$`, "i").exec(item) - return m ? unquoteName(m[1]!) : null + if (!m || NOT_TYPE.test(m[2]!.replace(new RegExp(QUOTED, "g"), " "))) return null + return unquoteName(m[1]!) } +/** Words that make the rest of an item more than a type (`amount::numeric is not null`, `amount::int as x`). */ +const NOT_TYPE = + /(? o.from === e.from && o.fromCol === e.fromCol && o.to === name)) e.to = name } diff --git a/packages/opencode/test/altimate/column-answer.test.ts b/packages/opencode/test/altimate/column-answer.test.ts index 8112e1d8a2..89e98248c7 100644 --- a/packages/opencode/test/altimate/column-answer.test.ts +++ b/packages/opencode/test/altimate/column-answer.test.ts @@ -1268,7 +1268,9 @@ describe("both tools: names at the edges", () => { test("on Postgres and Redshift a cast of a column is named after it, so the trace goes on", () => { for (const dialect of ["postgres", "redshift"]) { - for (const cast of ["amount::numeric", "cast(amount as numeric)", "amount::int[]", "cast(amount as int[])", `amount::"numeric"`, "amount::numeric::int"]) { + for (const cast of ["amount::numeric", "cast(amount as numeric)", "amount::int[]", "cast(amount as int[])", `amount::"numeric"`, "amount::numeric::int", + "amount::int[3]", "amount::pg_catalog.numeric", "amount::double precision", "amount::numeric(10, 2)"]) { + if (dialect === "redshift" && cast.includes("[")) continue // Redshift has no array types const path = write(nodes(dialect, [ model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), model("r", ["orders"], `select order_id, ${cast} from ${T("orders")}`), @@ -1300,6 +1302,16 @@ describe("both tools: names at the edges", () => { expect(carried(columnAnswer(duck, "orders", "amount"))).toEqual(["r._col_1"]) }) + test("an expression that only starts with a cast isn't named after the column, and a _col_N the SQL writes stays", () => { + for (const item of ["amount::numeric is not null", "amount::int in (1, 2)", "cast(amount as int) between 1 and 2", "amount::numeric as _col_1"]) { + const path = write(nodes("postgres", [ + model("orders", [], "select 1 as order_id, 2.5 as amount", ["order_id", "amount"]), + model("r", ["orders"], `select order_id, ${item} from ${T("orders")}`), + ])) + expect(carried(columnAnswer(path, "orders", "amount"))).toEqual(["r._col_1"]) + } + }) + test("DuckDB names a repeated output from a join val_1, and a select * child lists it", () => { const path = write(nodes("duckdb", [ model("a", [], "select 1 as id, 2 as val", ["id", "val"]), From 146922093dcabfd8d79fc87c663dce6a978cbc87 Mon Sep 17 00:00:00 2001 From: "Ralph Sto. Domingo" Date: Mon, 16 Nov 2026 20:57:55 +0800 Subject: [PATCH 5/5] fix: a column named all_* or distinct_* is not a SELECT modifier - `select allocation::numeric` no longer reads as `all ocation`, so Postgres/Redshift don't rename it `ocation` or throw "has no column allocation" - DuckDB repeat names follow list order: `id, id, amount as id_1` is `id, id_1, id_1_1`, with no `id_2` Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VEmKQwEiHMwQ7BdwgPo9sb --- .../altimate/native/dbt/project-lineage.ts | 30 +++++++++++-------- .../test/altimate/column-answer.test.ts | 22 ++++++++++++++ 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/project-lineage.ts b/packages/opencode/src/altimate/native/dbt/project-lineage.ts index f6d15a72c3..ab22c8dc07 100644 --- a/packages/opencode/src/altimate/native/dbt/project-lineage.ts +++ b/packages/opencode/src/altimate/native/dbt/project-lineage.ts @@ -136,7 +136,8 @@ export function selectItems(stripped: string): string[] { if (start < 0) { if (boundary && word === "select") { start = i + 6 - const lead = /^\s+(?:distinct\s+on\s*\([^)]*\)|distinct|all|top\s+\d+)/i.exec(text.slice(start)) + // A whole word only: `allocation` and `distinct_users` are column names, not modifiers. + const lead = /^\s+(?:distinct\s+on\s*\([^)]*\)|distinct|all|top\s+\d+)(?![\w$])/i.exec(text.slice(start)) if (lead) start += lead[0].length from = start i = start - 1 @@ -215,18 +216,22 @@ function columnsTemplates(stripped: string): Set { return out } -/** DuckDB's names for a SELECT list's repeated outputs: the second `val` is `val_1`, the third `val_2`. */ -function duckdbRepeats(outputs: string[], taken: Set): string[] { - const seen = new Map() +/** + * The names DuckDB gives a SELECT list's repeated outputs, in list order: an output whose name is already used takes + * the first free `_N` (`val, val, val` is `val, val_1, val_2`; `id, id, amount as id_1` is `id, id_1, id_1_1`). + */ +function duckdbRepeats(outputs: string[]): string[] { + const used = new Set() const out: string[] = [] for (const raw of outputs) { const name = raw.toLowerCase() - const n = seen.get(name) ?? 0 - seen.set(name, n + 1) - if (n === 0) continue - let k = n - while (taken.has(`${name}_${k}`)) k++ - taken.add(`${name}_${k}`) + if (!used.has(name)) { + used.add(name) + continue + } + let k = 1 + while (used.has(`${name}_${k}`)) k++ + used.add(`${name}_${k}`) out.push(`${name}_${k}`) } return out @@ -641,9 +646,8 @@ export class ProjectLineage { const outputs = sql ? this.outputColumns(sql).filter(real) : [] for (const c of outputs) add(c) // DuckDB keeps a repeated output under a new name (`val`, `val_1`): a `select *` over it reads both. Only - // names the SQL makes are taken; a documented `val_1` is that rename. - const made = new Set([...edges.map((e) => e.to), ...outputs].filter(real).map((name) => name.toLowerCase())) - if (this.dialect === "duckdb") for (const c of duckdbRepeats(outputs, made)) add(c) + // the SELECT list decides the names; a documented `val_1` is that rename. + if (this.dialect === "duckdb") for (const c of duckdbRepeats(outputs)) add(c) } const out = [...found.values()] this.columnMemo.set(id, out) diff --git a/packages/opencode/test/altimate/column-answer.test.ts b/packages/opencode/test/altimate/column-answer.test.ts index 89e98248c7..c2bb551bdc 100644 --- a/packages/opencode/test/altimate/column-answer.test.ts +++ b/packages/opencode/test/altimate/column-answer.test.ts @@ -1302,6 +1302,20 @@ describe("both tools: names at the edges", () => { expect(carried(columnAnswer(duck, "orders", "amount"))).toEqual(["r._col_1"]) }) + test("a column name that starts with all or distinct isn't read as a SELECT modifier", () => { + for (const name of ["allocation", "distinct_users", "all_time"]) { + const path = write(nodes("postgres", [ + model("f", [], `select 2.5 as ${name}`, [name]), + model("r", ["f"], `with q as (select * from ${T("f")}) select ${name}::numeric from q`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + expect(() => columnReach(path, "r", name)).not.toThrow() + expect(() => columnAnswer(path, "r", name)).not.toThrow() + const lineage = dbtLineage({ manifest_path: path, model: "child" }) + expect(Object.keys((lineage.raw_lineage as any).column_dict ?? {})).toContain(name) + } + }) + test("an expression that only starts with a cast isn't named after the column, and a _col_N the SQL writes stays", () => { for (const item of ["amount::numeric is not null", "amount::int in (1, 2)", "cast(amount as int) between 1 and 2", "amount::numeric as _col_1"]) { const path = write(nodes("postgres", [ @@ -1330,6 +1344,14 @@ describe("both tools: names at the edges", () => { ])) const listed = dbtLineage({ manifest_path: documented, model: "child" }) expect(Object.keys((listed.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "val", "val_1"]) + // In list order, as DuckDB names them: `id, id, amount as id_1` is `id, id_1, id_1_1`, with no `id_2`. + const clash = write(nodes("duckdb", [ + model("f", [], "select 1 as id, 2.5 as amount", ["id", "amount"]), + model("r", ["f"], `select id, id, amount as id_1 from ${T("f")}`), + model("child", ["r"], `select * from ${T("r")}`), + ])) + const named = dbtLineage({ manifest_path: clash, model: "child" }) + expect(Object.keys((named.raw_lineage as any).column_dict ?? {}).sort()).toEqual(["id", "id_1", "id_1_1"]) }) test("the text line says how many may-carry models are only not traced, without suggesting no effect", () => {