diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index 7a03aeb906..cb53c0ec7b 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -347,6 +347,26 @@ export namespace AltimateApi { // altimate_change end } + /** + * Calls an Altimate API endpoint with the stored credentials (Bearer key and x-tenant headers) and + * returns the parsed JSON body. Endpoint functions in other files go through this. Fails with a + * message naming the credentials file when none are stored or they can't be read. + */ + export async function apiRequest(method: string, endpoint: string, body?: unknown): Promise { + let creds: AltimateCredentials + try { + creds = await getCredentials() + } catch (e) { + const p = credentialsPath() + if (!(await Filesystem.exists(p))) + throw new Error( + `No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${p}).`, + ) + throw new Error(`The Altimate credentials in ${p} can't be used: ${e instanceof Error ? e.message : String(e)}`) + } + return request(creds, method, endpoint, body) + } + export async function listDatamates(creds?: AltimateCredentials) { creds ??= await getCredentials() const data = await request(creds, "GET", "/datamates/") diff --git a/packages/opencode/src/altimate/tool-cli/allow-list.ts b/packages/opencode/src/altimate/tool-cli/allow-list.ts index 33b452d75c..0bb4f18473 100644 --- a/packages/opencode/src/altimate/tool-cli/allow-list.ts +++ b/packages/opencode/src/altimate/tool-cli/allow-list.ts @@ -35,10 +35,19 @@ import { AltimateCoreTestgenTool } from "../tools/altimate-core-testgen" import { AltimateCoreTrackLineageTool } from "../tools/altimate-core-track-lineage" import { AltimateCoreValidateTool } from "../tools/altimate-core-validate" import { DataDiffTool } from "../tools/data-diff" +import { DbtCostEstimateTool } from "../tools/dbt-cost-estimate" +import { DbtCteProfileTool } from "../tools/dbt-cte-profile" +import { DbtDeferStateTool } from "../tools/dbt-defer-state" +import { DbtDocsCoverageTool } from "../tools/dbt-docs-coverage" +import { DbtHealthTool } from "../tools/dbt-health" import { DbtLineageTool } from "../tools/dbt-lineage" import { DbtManifestTool } from "../tools/dbt-manifest" +import { DbtMapRelationsTool } from "../tools/dbt-map-relations" import { DbtProfilesTool } from "../tools/dbt-profiles" +import { DbtRunResultsTool } from "../tools/dbt-run-results" +import { DbtSqlGraphTool } from "../tools/dbt-sql-graph" import { DbtUnitTestGenTool } from "../tools/dbt-unit-test-gen" +import { DocsShareCommentsTool } from "../tools/docs-share-comments" import { FinopsAnalyzeCreditsTool } from "../tools/finops-analyze-credits" import { FinopsExpensiveQueriesTool } from "../tools/finops-expensive-queries" import { FinopsQueryHistoryTool } from "../tools/finops-query-history" @@ -106,10 +115,19 @@ export const EXPOSED: Record = { altimate_core_track_lineage: AltimateCoreTrackLineageTool, altimate_core_validate: AltimateCoreValidateTool, data_diff: DataDiffTool, + dbt_cost_estimate: DbtCostEstimateTool, + dbt_cte_profile: DbtCteProfileTool, + dbt_defer_state: DbtDeferStateTool, + dbt_docs_coverage: DbtDocsCoverageTool, + dbt_health: DbtHealthTool, dbt_lineage: DbtLineageTool, dbt_manifest: DbtManifestTool, + dbt_map_relations: DbtMapRelationsTool, dbt_profiles: DbtProfilesTool, + dbt_run_results: DbtRunResultsTool, + dbt_sql_graph: DbtSqlGraphTool, dbt_unit_test_gen: DbtUnitTestGenTool, + docs_share_comments: DocsShareCommentsTool, finops_analyze_credits: FinopsAnalyzeCreditsTool, finops_expensive_queries: FinopsExpensiveQueriesTool, finops_query_history: FinopsQueryHistoryTool, diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts new file mode 100644 index 0000000000..7a271cab96 --- /dev/null +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -0,0 +1,352 @@ +/** + * CTE detection and count-query building for the CTE profiler. + * + * Works on the raw model text (Jinja included) with string offsets. The scanner + * skips SQL line/block comments, Jinja comments and string literals, and + * understands quoted CTE names and CTE column lists. + */ + +export interface CteInfo { + /** Identifier as written, including a trailing column list such as `my_cte (id, name)`. */ + name: string + /** Offset where the name starts. */ + nameStart: number + /** Offset just after the name (and its column list). */ + nameEnd: number + /** Offset of the first character after the CTE body's opening parenthesis. */ + queryStart: number + /** Offset of the CTE body's closing parenthesis. */ + queryEnd: number + /** Order of the CTE within its WITH clause. */ + index: number + /** Offset of the WITH keyword that opens this CTE's clause. */ + withClauseStart: number +} + +/** Position just past the string literal, quoted identifier or comment (`--`, block, Jinja) that starts at `pos`; `pos` itself when none does. An unterminated one runs to the end of the text. */ +function skipLiteral(text: string, pos: number): number { + const char = text[pos] + const next = text[pos + 1] + if (char === "'" || char === '"' || char === "`") { + let i = pos + 1 + while (i < text.length) { + if (text[i] === char) { + // A doubled quote is an escaped quote. + if (text[i + 1] === char) i += 2 + else return i + 1 + } else i++ + } + return text.length + } + if (char === "$" && !/[\p{L}\p{N}_$]/u.test(text[pos - 1] ?? " ")) { + // A dollar-quoted string (`$$ … $$`, `$tag$ … $tag$`) runs to its matching tag. + const tag = /^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/.exec(text.substring(pos, pos + 70))?.[0] + if (tag) { + const end = text.indexOf(tag, pos + tag.length) + return end === -1 ? text.length : end + tag.length + } + } + const close = char === "-" && next === "-" ? "\n" : char === "/" && next === "*" ? "*/" : char === "{" && next === "#" ? "#}" : null + if (close === null) return pos + const end = text.indexOf(close, pos + 2) + if (end === -1) return text.length + return close === "\n" ? end : end + close.length +} + +/** Position of the first character at or after `pos` that is not whitespace, a comment, or (when `jinja`) a Jinja `{% %}` tag. */ +function skipTrivia(text: string, pos: number, jinja: boolean): number { + while (pos < text.length) { + if (/\s/.test(text[pos])) { + pos++ + continue + } + const end = skipLiteral(text, pos) + if (end !== pos && !/['"`]/.test(text[pos])) { + pos = end + continue + } + if (jinja && text[pos] === "{" && text[pos + 1] === "%") { + pos = skipJinjaBlock(text, pos) + continue + } + return pos + } + return pos +} + +/** Offset of the `)` that closes the `(` at `openParenPos`, or -1 when it is never closed. */ +function findMatchingClosingParen(text: string, openParenPos: number): number { + let depth = 1 + let pos = openParenPos + 1 + while (pos < text.length) { + const end = skipLiteral(text, pos) + if (end !== pos) { + pos = end + continue + } + if (text[pos] === "(") depth++ + else if (text[pos] === ")" && --depth === 0) return pos + pos++ + } + return -1 +} + +interface JinjaTag { + /** Tag name (`if`, `set`, `macro`…), without the `end` prefix. */ + name: string + /** The tag closes a block (`{% endif %}`). */ + closes: boolean + /** The tag opens a block: not an `end` tag, `else`/`elif`, or an inline `{% set x = … %}`. */ + opens: boolean + /** Offset just past the tag. */ + end: number +} + +const BLOCK_TAGS = new Set(["if", "for", "macro", "call", "filter", "with", "set", "raw", "block"]) + +/** The block tag of the `{% … %}` that spans `at` to `close` (the offset of its `%}`); undefined for any other tag. */ +function parseTag(text: string, at: number, close: number): JinjaTag | undefined { + const head = /^\{%[-+]?\s*(end)?(\w+)/.exec(text.substring(at, Math.min(close + 2, at + 200))) + if (!head || !BLOCK_TAGS.has(head[2])) return undefined + const closes = head[1] !== undefined + // `{% set x = … %}` is an inline set; `{% set x %}` and `{% set x | filter %}` open a block. + const inline = head[2] === "set" && /^\{%[-+]?\s*set\s+[\w.,\s()]+=(?!=)/.test(text.substring(at, close + 2)) + return { name: head[2], closes, opens: !closes && !inline, end: close + 2 } +} + +/** + * The Jinja statement tags of `text` from `from`, in order, in one linear pass. Comments (`{# #}`) and the + * inside of `{% raw %}` are not tags; an unterminated comment or tag ends the text. + */ +function* jinjaTags(text: string, from: number): Generator { + let pos = from + for (;;) { + const at = text.indexOf("{", pos) + if (at === -1 || at + 1 >= text.length) return + const kind = text[at + 1] + if (kind !== "#" && kind !== "%") { + pos = at + 1 + continue + } + const close = text.indexOf(kind === "#" ? "#}" : "%}", at + 2) + if (close === -1) return + pos = close + 2 + if (kind === "#") continue + const tag = parseTag(text, at, close) + if (!tag) continue + if (tag.name === "raw" && tag.opens) { + // Nothing inside `{% raw %}` is Jinja: the block runs to its end tag. + const end = /\{%[-+]?\s*endraw\b/g + end.lastIndex = pos + const found = end.exec(text) + if (!found) { + yield { name: "raw", closes: false, opens: true, end: text.length } + return + } + const endClose = text.indexOf("%}", found.index) + pos = endClose === -1 ? text.length : endClose + 2 + yield { name: "raw", closes: false, opens: false, end: pos } + continue + } + yield tag + } +} + +/** + * Position just past the Jinja tag at `pos`, and past its whole block when it opens one whose text + * is not the model's own SQL (`{% set x %}…{% endset %}`, `{% call %}`, `{% macro %}`): a query + * written there is a string for dbt to run, not a CTE of the model. + */ +function skipJinjaBlock(text: string, pos: number): number { + const close = text.indexOf("%}", pos + 2) + if (close === -1) return text.length + const first = parseTag(text, pos, close) + if (!first || !first.opens || (first.name !== "set" && first.name !== "call" && first.name !== "macro")) return close + 2 + // Blocks of the same kind nest: the block ends at the end tag that matches this one. + let depth = 1 + for (const tag of jinjaTags(text, close + 2)) { + if (tag.name !== first.name) continue + if (tag.closes) depth-- + else if (tag.opens) depth++ + if (depth === 0) return tag.end + } + return text.length +} + +/** Every WITH keyword outside comments, strings and parentheses: a WITH inside a CTE body or a subquery belongs to that query, not to the model. */ +function findTopLevelWithKeywords(text: string): number[] { + const positions: number[] = [] + const keyword = /with\b/iy + let depth = 0 + let pos = 0 + while (pos < text.length) { + const end = skipLiteral(text, pos) + if (end !== pos) { + pos = end + continue + } + const char = text[pos] + if (char === "{" && text[pos + 1] === "%") { + pos = skipJinjaBlock(text, pos) + continue + } + if (char === "(") depth++ + else if (char === ")") depth = Math.max(0, depth - 1) + else if (depth === 0 && (char === "w" || char === "W") && !/[A-Za-z0-9_$]/.test(text[pos - 1] ?? " ")) { + keyword.lastIndex = pos + if (keyword.test(text)) { + positions.push(pos) + pos += 4 + continue + } + } + pos++ + } + return positions +} + +/** Offset just past the identifier (`a`, `"a""b"`, `` `a` ``, `[a]`, `db.schema.a`) that starts at `pos`; -1 when none does. */ +function readIdentifier(text: string, pos: number): number { + let end = pos + for (;;) { + const char = text[end] + if (char === '"' || char === "`") { + const close = skipLiteral(text, end) + if (close === end || text[close - 1] !== char || close - end < 2) return -1 + end = close + } else if (char === "[") { + const bracketed = /\[(?:[^\]\n]|\]\]){1,256}\]/y + bracketed.lastIndex = end + if (!bracketed.test(text)) return -1 + end = bracketed.lastIndex + } else { + const bare = /[\p{L}_][\p{L}\p{N}_$]*/uy + bare.lastIndex = end + if (!bare.test(text)) return -1 + end = bare.lastIndex + } + if (text[end] !== ".") return end + end++ + } +} + +/** Offset of the `(` that opens a CTE body after `AS [NOT] [MATERIALIZED]` starting at `pos`; -1 when the text there is not such a header. */ +function skipHeader(text: string, pos: number): number { + let at = pos + const word = (w: string) => { + const re = new RegExp(`${w}\\b`, "iy") + re.lastIndex = at + if (!re.test(text)) return false + at = skipTrivia(text, at + w.length, true) + return true + } + if (!word("as")) return -1 + if (word("not")) { + if (!word("materialized")) return -1 + } else word("materialized") + return text[at] === "(" ? at : -1 +} + +/** The CTEs of the WITH clause that opens at `withPos`: each `name [(columns)] AS (body)`, separated by commas, up to the first thing that is not one. */ +function parseWithClause(text: string, withPos: number, ctes: CteInfo[]): void { + let pos = skipTrivia(text, withPos + 4, true) + const keyword = /recursive\b/iy + keyword.lastIndex = pos + if (keyword.test(text)) pos = skipTrivia(text, pos + "recursive".length, true) + + for (let index = 0; ; index++) { + const nameStart = pos + const identifierEnd = readIdentifier(text, nameStart) + if (identifierEnd === -1) return + let nameEnd = identifierEnd + let name = text.substring(nameStart, identifierEnd) + pos = skipTrivia(text, identifierEnd, false) + if (text[pos] === "(") { + const close = findMatchingClosingParen(text, pos) + if (close === -1) return + // The name as written plus its column list; a comment between them is left out. + name += ` ${text.substring(pos, close + 1)}` + nameEnd = close + 1 + pos = skipTrivia(text, nameEnd, false) + } + // `AS [NOT] [MATERIALIZED] (`, with comments or Jinja tags allowed between the words. + const open = skipHeader(text, pos) + if (open === -1) return + const close = findMatchingClosingParen(text, open) + if (close === -1) return + ctes.push({ + name, + nameStart, + nameEnd, + queryStart: open + 1, + queryEnd: close, + index, + withClauseStart: withPos, + }) + pos = skipTrivia(text, close + 1, true) + if (text[pos] !== ",") return + pos = skipTrivia(text, pos + 1, true) + } +} + +/** Find every CTE of the model's own WITH clauses, in source order. A WITH inside a CTE body or a subquery is not listed: its CTEs cannot be counted without the query around them. */ +export function detectCtes(text: string): CteInfo[] { + const ctes: CteInfo[] = [] + for (const withPos of findTopLevelWithKeywords(text)) parseWithClause(text, withPos, ctes) + return ctes +} + +/** Drop a trailing column list: `my_cte (id, name)` -> `my_cte`. */ +export function stripColumnList(rawName: string): string { + const end = readIdentifier(rawName, 0) + return end === -1 || !rawName.substring(end).trimStart().startsWith("(") ? rawName.trim() : rawName.substring(0, end) +} + +export function quoteSqlIdentifier(identifier: string): string { + if (identifier.match(/^["'`[]/) || identifier.includes(".")) return identifier + if (!identifier.match(/^[\p{L}_][\p{L}\p{N}_$]*$/u)) return `"${identifier}"` + return identifier +} + +/** Name of the first Jinja block (`if`, `for`, `macro`, `call`, `filter`, `with`, `raw`, or `set` as a block) opened in `text` and never closed there; undefined when every block is closed. Tags inside Jinja comments and `{% raw %}` do not count. */ +export function unclosedJinjaBlock(text: string): string | undefined { + const open: string[] = [] + const counts = new Map() + for (const tag of jinjaTags(text, 0)) { + if (tag.closes) { + // An end tag closes the nearest open block of its kind, and any block opened inside it. + if (!counts.get(tag.name)) continue + for (let top = open.pop(); top !== undefined; top = open.pop()) { + counts.set(top, counts.get(top)! - 1) + if (top === tag.name) break + } + } else if (tag.opens) { + open.push(tag.name) + counts.set(tag.name, (counts.get(tag.name) ?? 0) + 1) + } + } + return open[0] +} + +/** + * `SELECT COUNT(*)` over one CTE: the model's text up to the end of that CTE (dbt configs, variables, + * Jinja between CTEs and every earlier CTE of the WITH clause, as written), so the query compiles on its own. + * Undefined for a CTE of a second WITH clause in the same file, which has no standalone query. + */ +export function buildCountQuery(text: string, ctes: CteInfo[], target: CteInfo): string | undefined { + // The text before a later WITH holds the earlier clause's whole query: nothing valid stands alone there. + if (ctes.some((c) => c.withClauseStart < target.withClauseStart)) return undefined + if (target.queryStart <= target.nameStart) return undefined + const head = text.substring(0, target.queryEnd + 1).trim() + return `${head}\nSELECT COUNT(*) AS _profile_count FROM ${quoteSqlIdentifier(stripColumnList(target.name))}` +} + +/** + * Read the single COUNT(*) value by position: Snowflake and Oracle upper-case + * the alias, so a lookup by name would report 0 rows. + */ +export function extractRowCount(data: Record[] | undefined): number { + if (!data || data.length === 0) return 0 + const first = Object.values(data[0])[0] + return typeof first === "number" ? first : Number(first) || 0 +} diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts new file mode 100644 index 0000000000..d73dd7e9c5 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -0,0 +1,101 @@ +import fs from "fs" +import path from "path" +import type { Tool } from "../../tool/tool" +import { projectRoot } from "../native/connections/registry" + +/** Why a tool may not use the dbt adapter. */ +export type DbtAdapterRefusal = "read_only" | "other_project" | "no_project" | "not_started" | "not_loaded" + +/** The project's dbt adapter, or why this call may not use it (`reason` is a sentence for the user). */ +export type DbtAdapterAccess = + | { adapter: any; refusal?: undefined; reason?: undefined } + | { adapter?: undefined; refusal: DbtAdapterRefusal; reason: string } + +/** Test seams. */ +export const dbtAdapterAccessInternals = { + /** How long a call waits for the dbt adapter to start before it gives up; the start goes on for the next call. */ + adapterStartTimeoutMs: 120_000, +} + +/** + * The dbt adapter for a tool call, which every adapter-backed tool goes through. Starting the adapter + * runs dbt deps and writes target/, and compiling renders Jinja, which can run SQL: a read-only call + * (no write permission, or plan mode) never starts or uses it. Nor does a session in a dbt project the + * adapter isn't set up for. The wait for the adapter to start is bounded and follows the call's abort. + */ +export async function dbtAdapterFor(ctx: Pick): Promise { + const signal = ctx.abort + if (ctx.extra?.forceReadOnly === true || ctx.agent === "plan") + return { + refusal: "read_only", + reason: "this needs the dbt adapter, which a read-only call doesn't start: re-run with --allow-write", + } + try { + const { getDbtAdapter, getDbtProjectRoot } = await import("../native/connections/register") + // The import yields: a call cancelled meanwhile starts nothing. + signal.throwIfAborted() + const project = await sessionProject() + signal.throwIfAborted() + if (project?.other) { + return { + refusal: "other_project", + reason: project.root + ? `the dbt adapter is set up for ${project.root}, not ${project.other}` + : `the dbt adapter isn't set up for ${project.other}`, + } + } + let stop = () => {} + let timer: ReturnType | undefined + const waited = new Promise((_, reject) => { + stop = () => reject(signal.reason) + signal.addEventListener("abort", stop, { once: true }) + const seconds = dbtAdapterAccessInternals.adapterStartTimeoutMs / 1000 + timer = setTimeout( + () => reject(new Error(`the dbt adapter didn't start within ${seconds} s`)), + dbtAdapterAccessInternals.adapterStartTimeoutMs, + ) + }) + const adapter = await Promise.race([getDbtAdapter(), waited]).finally(() => { + signal.removeEventListener("abort", stop) + clearTimeout(timer) + }) + if (!adapter) { + // No adapter also when the project exists but the adapter failed to start: say which. + const root = await getDbtProjectRoot() + const found = !!root && fs.existsSync(path.join(root, "dbt_project.yml")) + return found + ? { refusal: "not_started", reason: "the dbt adapter didn't start" } + : { refusal: "no_project", reason: "no dbt project here" } + } + if (adapter.isInitialized?.() === false) + return { refusal: "not_loaded", reason: "the dbt adapter didn't load the project" } + return { adapter } + } catch (e) { + if (signal.aborted) throw e + return { refusal: "not_started", reason: e instanceof Error ? e.message : String(e) } + } +} + +/** + * The adapter's project root and, when the session's directory is in another dbt project, that one + * (`other`, both links resolved). Undefined when the session's directory is in no dbt project: the + * adapter's own discovery (a dbt.json, or a walk up from the working directory) decides then. + */ +async function sessionProject(): Promise<{ root?: string; other?: string } | undefined> { + const { findProjectRoot } = await import("../../../../dbt-tools/src/config") + const found = findProjectRoot(projectRoot()) + if (!found) return undefined + const { getDbtProjectRoot } = await import("../native/connections/register") + const root = await getDbtProjectRoot() + const real = (p: string) => { + try { + return fs.realpathSync(p) + } catch { + return path.resolve(p) + } + } + const here = real(found) + // path.relative ignores case on win32. + if (!root) return { other: here } + return path.relative(real(root), here) ? { root: real(root), other: here } : { root } +} diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts new file mode 100644 index 0000000000..1b113f1994 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -0,0 +1,17 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtCostEstimateTool = Tool.define("dbt_cost_estimate", { + description: "Estimate the bytes a dbt model or a SQL query would scan, from a warehouse dry run (BigQuery).", + parameters: z.object({ + model: z.string().optional().describe("Model name to compile and estimate"), + sql: z.string().optional().describe("SQL (may use Jinja) to compile and estimate"), + }), + async execute() { + return { + title: "dbt_cost_estimate: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_cost_estimate is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts new file mode 100644 index 0000000000..bb23811d50 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -0,0 +1,230 @@ +import fs from "fs" +import path from "path" +import z from "zod" +import { Tool } from "../../tool/tool" +import { loadRawManifest, findModel } from "../native/dbt/helpers" +import { projectRoot } from "../native/connections/registry" +import { isQueryTimeoutError } from "@altimateai/drivers/types" +import { executeQuery } from "../native/query-execution" +import { dbtAdapterFor } from "./dbt-adapter-access" +import { buildCountQuery, detectCtes, extractRowCount, stripColumnList, unclosedJinjaBlock, type CteInfo } from "./cte-detect" + +/** Folders a model search never enters. */ +const SKIPPED_DIRS = new Set(["target", "dbt_packages", "dbt_modules", "node_modules", "logs", "venv", "site-packages"]) + +/** Deepest folder level the model search follows. */ +const MAX_SEARCH_DEPTH = 12 + +/** Largest model file the tool reads. */ +const MAX_MODEL_BYTES = 4 * 1024 * 1024 + +/** A CTE name without its column list or surrounding quotes, quote escapes undone. */ +function bareName(name: string): string { + const plain = stripColumnList(name) + const quote = plain[0] + if ((quote === '"' || quote === "`") && plain.length > 1 && plain.endsWith(quote)) return plain.slice(1, -1).replaceAll(quote + quote, quote) + if (quote === "[" && plain.endsWith("]")) return plain.slice(1, -1) + return plain +} + +function inside(root: string, file: string): boolean { + const rel = path.relative(root, file) + return rel !== "" && rel !== ".." && !rel.startsWith(".." + path.sep) && !path.isAbsolute(rel) +} + +function realpath(p: string): string { + try { + return fs.realpathSync(p) + } catch { + return path.resolve(p) + } +} + +/** Every `.sql` under the project, outside the build folders. */ +function findSqlFiles(root: string, name: string): string[] { + const found: string[] = [] + const walk = (dir: string, depth: number) => { + let entries: fs.Dirent[] + try { + entries = fs.readdirSync(dir, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + if (entry.name.startsWith(".")) continue + const full = path.join(dir, entry.name) + if (entry.isDirectory()) { + if (depth < MAX_SEARCH_DEPTH && !SKIPPED_DIRS.has(entry.name)) walk(full, depth + 1) + } else if (entry.isFile() && entry.name === `${name}.sql`) { + found.push(full) + } + } + } + walk(root, 0) + return found.sort() +} + +/** The model's source file: a project-relative or absolute `.sql` path, the manifest's record of a name or unique_id, or a search by name. */ +function modelFile(root: string, model: string): { file?: string; error?: string } { + const realRoot = realpath(root) + if (/\.sql$/i.test(model) || /[\\/]/.test(model)) { + const candidate = path.resolve(realRoot, model) + if (!/\.sql$/i.test(candidate)) return { error: `${model} is not a .sql file` } + if (!inside(realRoot, realpath(candidate))) return { error: `${model} is outside the dbt project` } + return fs.existsSync(candidate) && fs.statSync(candidate).isFile() ? { file: realpath(candidate) } : { error: `no file ${model} in the dbt project` } + } + try { + const manifest = loadRawManifest(path.join(realRoot, "target", "manifest.json")) + const own = /^name:\s*['"]?([\w-]+)/m.exec(fs.readFileSync(path.join(realRoot, "dbt_project.yml"), "utf-8"))?.[1] + let node = manifest?.nodes ? findModel(manifest.nodes, model) : null + if (node && !model.startsWith("model.")) { + // The project's own model of that name comes before a package's; versions of one model share a name and the manifest can't say which is meant. + const named = Object.values(manifest.nodes).filter((n) => n.resource_type === "model" && n.name === model) + const mine = named.filter((n) => !own || !n.package_name || n.package_name === own) + const pool = mine.length > 0 ? mine : named + if (pool.length > 1) return { error: `${model} matches ${pool.length} models (${pool.map((n) => n.unique_id).join(", ")}): pass the path of the one to profile` } + node = pool[0] + } + if (node?.original_file_path) { + // dbt records the path relative to the model's own package: only the project's models can be read here. + if (node.package_name && own && node.package_name !== own) return { error: `${model} belongs to the package ${node.package_name}: pass the path of its .sql file` } + const candidate = path.resolve(realRoot, node.original_file_path) + if (inside(realRoot, realpath(candidate)) && fs.existsSync(candidate)) return { file: realpath(candidate) } + } + } catch { + // A manifest that can't be read doesn't stop the search by name. + } + // A unique_id names one model of one package: only the manifest can say which file it is. + if (model.startsWith("model.")) return { error: `${model} is not in target/manifest.json: run dbt compile, or pass the model's name or path` } + const matches = findSqlFiles(realRoot, model) + if (matches.length === 1) return { file: realpath(matches[0]) } + if (matches.length === 0) return { error: `no model named ${model} in the dbt project` } + const rel = matches.map((m) => path.relative(realRoot, m)) + return { error: `${model} matches ${matches.length} files (${rel.join(", ")}): pass the path` } +} + +/** The dbt project of the session's directory; else the one the dbt adapter is set up for. */ +async function projectDir(): Promise { + const { findProjectRoot } = await import("../../../../dbt-tools/src/config") + const here = findProjectRoot(projectRoot()) + if (here) return here + const { getDbtProjectRoot } = await import("../native/connections/register") + return getDbtProjectRoot() +} + +function failure(title: string, output: string, extra: Record = {}) { + return { title: `dbt_cte_profile: ${title}`, metadata: { success: false, error: output, ...extra }, output } +} + +export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { + description: + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE.", + parameters: z.object({ + model: z + .string() + .describe("Model name, unique_id, or project-relative or absolute path to the model's .sql file (e.g. 'customers' or 'models/marts/customers.sql')"), + cte: z.string().optional().describe("CTE to time, by name; without it (and without `position`), the CTE list is returned"), + position: z + .number() + .int() + .min(0) + .optional() + .describe("The CTE's position in the list (0-based); needed when several CTEs share a name"), + }), + async execute(args, ctx) { + const root = await projectDir() + if (!root) return failure("no dbt project", "No dbt project here: run from a dbt project folder.") + const located = modelFile(root, args.model) + if (!located.file) return failure("model not found", located.error ?? `no model named ${args.model}`) + const size = fs.statSync(located.file).size + if (size > MAX_MODEL_BYTES) return failure("model too large", `${args.model} is ${size} bytes; the largest file read is ${MAX_MODEL_BYTES}.`) + const text = fs.readFileSync(located.file, "utf-8") + const ctes = detectCtes(text) + const file = path.relative(realpath(root), located.file) + + if (args.cte === undefined && args.position === undefined) { + // The CTEs come in source order: count the lines once, not once per CTE. + let line = 1 + let counted = 0 + const lineAt = (offset: number) => { + for (; counted < offset; counted++) if (text[counted] === "\n") line++ + return line + } + const list = ctes.map((c, position) => ({ + name: stripColumnList(c.name), + line: lineAt(c.nameStart), + index: c.index, + clause: c.withClauseStart, + position, + })) + const output = list.length + ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c) => `${c.position + 1}. ${c.name} (line ${c.line})`)].join("\n") + : `No CTEs in ${file}.` + return { + title: `dbt_cte_profile: ${list.length} CTE${list.length === 1 ? "" : "s"} in ${args.model}`, + metadata: { success: true, model: args.model, file, ctes: list }, + output, + } + } + + const names = ctes.map((c) => stripColumnList(c.name)).join(", ") + let target: CteInfo | undefined + if (args.position !== undefined) { + target = ctes[args.position] + if (!target) return failure("CTE not found", `${args.model} has ${ctes.length} CTEs: there is none at position ${args.position}.`, { model: args.model }) + if (args.cte !== undefined && bareName(target.name) !== bareName(args.cte) && bareName(target.name).toLowerCase() !== bareName(args.cte).toLowerCase()) + return failure("CTE not found", `The CTE at position ${args.position} of ${args.model} is ${stripColumnList(target.name)}, not ${args.cte}.`, { model: args.model }) + } else { + // The name as typed wins; letter case only decides when nothing matches it exactly. + const wanted = bareName(args.cte ?? "") + const exact = ctes.filter((c) => bareName(c.name) === wanted) + const matches = exact.length > 0 ? exact : ctes.filter((c) => bareName(c.name).toLowerCase() === wanted.toLowerCase()) + if (matches.length === 0) + return failure("CTE not found", `${args.model} has no CTE named ${args.cte}${names ? `; it has: ${names}` : ""}.`, { model: args.model }) + if (matches.length > 1) { + const at = matches.map((c) => ctes.indexOf(c)).join(", ") + return failure("CTE ambiguous", `${args.model} has ${matches.length} CTEs that match ${args.cte} (positions ${at}): pass position.`, { model: args.model }) + } + target = matches[0] + } + const query = buildCountQuery(text, ctes, target) + const name = stripColumnList(target.name) + if (!query) { + const block = unclosedJinjaBlock(text.substring(0, target.withClauseStart)) + return failure( + "no query", + block + ? `${name} sits in a WITH clause of ${args.model} that is one branch of a Jinja {% ${block} %} block, so it has no query of its own to count.` + : `${name} is in a second WITH clause of ${args.model}, which has no query of its own to count.`, + { model: args.model, cte: name }, + ) + } + const block = unclosedJinjaBlock(query) + if (block) return failure("no query", `${name} is inside a Jinja {% ${block} %} block of ${args.model}, which cannot be cut off at that CTE and stay valid: compile the model and profile the compiled SQL, or move the WITH out of the block.`, { model: args.model, cte: name }) + + const access = await dbtAdapterFor(ctx) + if (!access.adapter) return failure("not run", access.reason ?? "the dbt adapter is not available", { model: args.model, cte: name, refusal: access.refusal }) + const adapter = access.adapter + ctx.abort.throwIfAborted() + const started = performance.now() + try { + // The adapter has no cancel of its own: the run's timeout, budget and abort end the wait, not the query. + const result: { data?: Record[] } = await executeQuery({ type: "dbt" }, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => + adapter.immediatelyExecuteSQLWithLimit(query, `cte_profiler_${name}`, 1), + ) + const queryMs = Math.round(performance.now() - started) + const rows = extractRowCount(result?.data) + return { + title: `dbt_cte_profile: ${name} ${queryMs} ms`, + metadata: { success: true, model: args.model, cte: name, index: target.index, position: ctes.indexOf(target), queryMs, rows }, + output: `${name}: ${queryMs} ms, ${rows} row${rows === 1 ? "" : "s"}`, + } + } catch (e) { + if (ctx.abort.aborted) throw e + const message = e instanceof Error ? e.message : String(e) + // A timeout gives up the wait, not the query: it may still be running on the warehouse. + const timedOut = isQueryTimeoutError(e) + return failure(timedOut ? "timed out" : "query failed", `The count query for ${name} ${timedOut ? "timed out and may still be running on the warehouse" : "failed"}: ${message}`, { model: args.model, cte: name, ...(timedOut && { timedOut }) }) + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-defer-state.ts b/packages/opencode/src/altimate/tools/dbt-defer-state.ts new file mode 100644 index 0000000000..9df344ec1b --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-defer-state.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtDeferStateTool = Tool.define("dbt_defer_state", { + description: "dbt Power User's defer-to-production setting for a dbt project, and the profile's default target.", + parameters: z.object({ + project_dir: z.string().describe("dbt project folder"), + }), + async execute() { + return { + title: "dbt_defer_state: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_defer_state is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts b/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts new file mode 100644 index 0000000000..8990306b72 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts @@ -0,0 +1,21 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtDocsCoverageTool = Tool.define("dbt_docs_coverage", { + description: + "Documentation coverage of a dbt model: which output columns have a description, and whether the model has one.", + parameters: z.object({ + model: z.string().describe("Model name"), + manifest_path: z + .string() + .optional() + .describe("manifest.json to read (default: the dbt project's target/manifest.json)"), + }), + async execute() { + return { + title: "dbt_docs_coverage: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_docs_coverage is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-health.ts b/packages/opencode/src/altimate/tools/dbt-health.ts new file mode 100644 index 0000000000..096e2bc3f4 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-health.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtHealthTool = Tool.define("dbt_health", { + description: "DataPilot project health insights for the dbt project.", + parameters: z.object({ + config_path: z.string().optional().describe("DataPilot config file to use"), + }), + async execute() { + return { + title: "dbt_health: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_health is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-map-relations.ts b/packages/opencode/src/altimate/tools/dbt-map-relations.ts new file mode 100644 index 0000000000..f949274620 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-map-relations.ts @@ -0,0 +1,21 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtMapRelationsTool = Tool.define("dbt_map_relations", { + description: "Map each table a raw SQL query reads to the dbt ref() or source() for it, from the manifest.", + parameters: z.object({ + sql: z.string().describe("Raw SQL to map"), + dialect: z.string().optional().describe("SQL dialect of the query"), + manifest_path: z + .string() + .optional() + .describe("manifest.json to match against (default: the dbt project's target/manifest.json)"), + }), + async execute() { + return { + title: "dbt_map_relations: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_map_relations is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-run-results.ts b/packages/opencode/src/altimate/tools/dbt-run-results.ts new file mode 100644 index 0000000000..c6f2cf396c --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-run-results.ts @@ -0,0 +1,19 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtRunResultsTool = Tool.define("dbt_run_results", { + description: "The latest dbt run's results from target/run_results.json: command, duration and each node's status.", + parameters: z.object({ + project_dir: z + .string() + .optional() + .describe("dbt project folder (default: the dbt project found from the working directory)"), + }), + async execute() { + return { + title: "dbt_run_results: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_run_results is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts new file mode 100644 index 0000000000..0c4db3c980 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtSqlGraphTool = Tool.define("dbt_sql_graph", { + description: "A dbt model's SQL as a graph: its CTEs, joins and filters, and the edges between them.", + parameters: z.object({ + model: z.string().describe("Model name"), + }), + async execute() { + return { + title: "dbt_sql_graph: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_sql_graph is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/docs-share-comments.ts b/packages/opencode/src/altimate/tools/docs-share-comments.ts new file mode 100644 index 0000000000..8963268cdd --- /dev/null +++ b/packages/opencode/src/altimate/tools/docs-share-comments.ts @@ -0,0 +1,20 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DocsShareCommentsTool = Tool.define("docs_share_comments", { + description: "Comment threads on shared dbt docs: list open threads, reply to one, or resolve one.", + parameters: z.object({ + action: z.enum(["list", "reply", "resolve"]).describe("What to do"), + project: z.string().optional().describe("dbt project name the shared docs belong to"), + share_id: z.number().int().optional().describe("Shared docs id"), + group_id: z.number().int().optional().describe("Comment thread id"), + text: z.string().optional().describe("Reply text"), + }), + async execute() { + return { + title: "docs_share_comments: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "docs_share_comments is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts new file mode 100644 index 0000000000..489f510eef --- /dev/null +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -0,0 +1,1636 @@ +import { describe, expect, it } from "bun:test" +import { + buildCountQuery, + detectCtes as detectCtesInText, + extractRowCount, + quoteSqlIdentifier, + stripColumnList, + unclosedJinjaBlock, + type CteInfo, +} from "../../src/altimate/tools/cte-detect" + +// Detector cases ported from the dbt Power User CTE detector suite. A "document" is the model text. +describe("detectCtes", () => { + const detectCtes = (document: string): CteInfo[] => detectCtesInText(document) + + const assertCTE = (cte: CteInfo, expectedName: string, expectedIndex: number, _description?: string) => { + expect(cte.name).toBe(expectedName) + expect(cte.index).toBe(expectedIndex) + expect(typeof cte.nameStart).toBe("number") + expect(typeof cte.queryStart).toBe("number") + expect(typeof cte.withClauseStart).toBe("number") + } + + describe("Basic CTE Detection", () => { + it("should detect single CTE correctly with proper ranges", () => { + const sql = "with my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with simple select and ID column") + }) + + it("should detect multiple CTEs correctly with proper indexing", () => { + const sql = `with + first_cte as (select 1 as id), + second_cte as (select 2 as id), + third_cte as (select 3 as id) + select * from third_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "first_cte", 0, "CTE with simple select") + assertCTE(ctes[1], "second_cte", 1, "CTE with different ID value") + assertCTE(ctes[2], "third_cte", 2, "CTE with third ID value") + }) + + it("should handle CTEs with column lists", () => { + const sql = "with my_cte (id, name) as (select 1, 'test') select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (id, name)", 0, "CTE with explicit column list specification") + }) + + it("should handle quoted CTE names", () => { + const sql = `with "my cte" as (select 1 as id) select * from "my cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], '"my cte"', 0, "CTE with double-quoted name containing space") + }) + }) + + describe("Comments in CTE Body", () => { + it("should handle line comments with unmatched quotes in CTE body", () => { + const sql = `with + cte_without_lens as ( + select * + from {{ ref('parent') }} + where + condition = true -- CTE doesn't have a lens + ), + cte_with_lens as ( + select * + from {{ ref('parent') }} + where + condition = true -- CTE has a 'lens' + ) + select * from cte_with_lens` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(2) + assertCTE(ctes[0], "cte_without_lens", 0, "CTE with line comment containing unmatched single quote") + assertCTE(ctes[1], "cte_with_lens", 1, "CTE with line comment containing matched single quotes") + }) + + it("should handle block comments with unmatched quotes in CTE body", () => { + const sql = `with + cte_with_block_comment as ( + select * + from {{ ref('parent') }} + where + condition = true /* This is a block comment with unmatched quote ' */ + ) + select * from cte_with_block_comment` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "cte_with_block_comment", 0, "CTE with block comment containing unmatched single quote") + }) + + it("should handle Jinja comments with unmatched quotes in CTE body", () => { + const sql = `with + cte_with_jinja_comment as ( + select * + from {{ ref('parent') }} + where + condition = true {# This is a Jinja comment with unmatched quote ' #} + ) + select * from cte_with_jinja_comment` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "cte_with_jinja_comment", 0, "CTE with Jinja comment containing unmatched single quote") + }) + }) + + describe("Comments Between CTE Name and AS", () => { + it("should handle single-line block comment between name and AS", () => { + const sql = "with source /* test */ as (select 1 as id) select * from source" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "source", 0, "CTE with single-line block comment between name and AS keyword") + // The important part is that CTE detection works correctly + }) + + it("should handle multi-line block comment between name and AS", () => { + const sql = `with source +/* this is +a multi-line comment +that should be ignored */ +as ( + select * from {{ ref('raw_orders') }} +) +select * from source` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "source", 0, "CTE with multi-line block comment between name and AS keyword") + // The important part is that CTE detection works correctly + }) + + it("should handle Jinja comment between name and AS", () => { + const sql = "with target {# some comment #} as (select 1 as id) select * from target" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "target", 0, "CTE with Jinja comment between name and AS keyword") + }) + + it("should handle line comment between name and AS (multiline)", () => { + const sql = `with multiline_cte -- this is a comment + as ( + select 4 as id + ) + select * from multiline_cte` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "multiline_cte", 0, "CTE with line comment on separate line before AS keyword") + }) + + it("should handle complex comment with quotes between name and AS", () => { + const sql = `with complex_comment /* more complex comment with 'quotes' */ as ( + select 5 as id + ) select * from complex_comment` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "complex_comment", 0, "CTE with block comment containing both single and double quotes") + }) + }) + + describe("Comments Before and Around CTE Names", () => { + it("should handle block comment before CTE name", () => { + const sql = "with /* comment before name */ my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with block comment before name") + }) + + it("should handle line comment before CTE name on separate line", () => { + const sql = `with + -- This is a comment before the CTE name + my_cte as ( + select 1 as id + ) + select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with line comment before name on separate line") + }) + + it("should handle Jinja comment before CTE name", () => { + const sql = "with {# comment before name #} my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with Jinja comment before name") + }) + + it("should handle comment immediately after WITH keyword", () => { + const sql = "with /* comment after WITH */ my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with block comment immediately after WITH keyword") + }) + + it("should handle line comment after WITH keyword", () => { + const sql = `with -- comment after WITH + my_cte as ( + select 1 as id + ) + select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with line comment after WITH keyword") + }) + + it("should handle multiple comments around CTE name", () => { + const sql = "with /* before */ my_cte /* after */ as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with block comments before and after name") + }) + + it("should handle mixed comment types around CTE name", () => { + const sql = `with -- line comment + /* block comment */ my_cte {# jinja comment #} as ( + select 1 as id + ) + select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with mixed comment types around name") + }) + + it("should handle comments in multi-CTE scenarios", () => { + const sql = `with + /* comment */ first_cte as ( + select 1 as id + ), + -- another comment + second_cte /* inline comment */ as ( + select 2 as id + ), + {# jinja comment #} third_cte as ( + select 3 as id + ) + select * from third_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "first_cte", 0, "First CTE with block comment before name") + assertCTE(ctes[1], "second_cte", 1, "Second CTE with line and inline comments") + assertCTE(ctes[2], "third_cte", 2, "Third CTE with Jinja comment before name") + }) + + it("should handle comments with special characters around CTE name", () => { + const sql = "with /* comment with $pecial ch@rs & symbols! */ my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with comment containing special characters") + }) + + it("should handle unterminated block comment before CTE name", () => { + const sql = "with /* unterminated comment my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + // The comment swallows the rest of the text, so there is no CTE to find. + expect(ctes).toEqual([]) + }) + + it("should handle sequential line and block comments between name and AS", () => { + const sql = `with source -- test +/* this is +a multi-line comment +that should be ignored */ +as ( + select * from {{ ref('raw_orders') }} +) +select * from renamed` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "source", 0, "CTE with sequential line and block comments between name and AS") + }) + }) + + describe("Complex Comment Scenarios", () => { + it("should handle CTE with mixed block and Jinja comments", () => { + const sql = `with +source /* test */ as( + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_orders') }} +), + +renamed as ( + select + id as order_id, + user_id as customer_id, + order_date, + status + from source +) + +select * from renamed` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(2) + assertCTE(ctes[0], "source", 0, "CTE with block comment after name and Jinja comment in body") + assertCTE(ctes[1], "renamed", 1, "CTE that references another CTE with column aliasing") + }) + + it("should handle CTE with multi-line block comment before AS keyword", () => { + const sql = `with source +/* this is +a multi-line comment +that should be ignored */ +as ( + select * from {{ ref('raw_orders') }} +), + +renamed as ( + select + id as order_id, + user_id as customer_id, + order_date, + status + from source +) + +select * from renamed` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(2) + assertCTE(ctes[0], "source", 0, "CTE with multi-line block comment spanning multiple lines before AS") + assertCTE(ctes[1], "renamed", 1, "CTE that selects from another CTE with column renaming") + }) + }) + + describe("Very Lengthy Comments with Quoted Names", () => { + it("should handle extremely long block comment before quoted CTE name", () => { + // Create a 1500+ character comment to test real-world lengthy comment scenarios + const longComment = + "/* " + + "This is an extremely long comment that might appear in real-world SQL files. ".repeat(25) + + "It contains multiple sentences and potentially problematic content like 'quoted strings', " + + "special characters !@#$%^&*()_+-=[]{}|;:,.<>?, and even problematic keywords " + + "that might confuse the parser. This comment also contains backslashes \\ and forward slashes // " + + "and other regex-problematic characters like .*+?^${}()|[] that could cause catastrophic backtracking " + + "in poorly designed regular expressions. The comment goes on and on longer. ".repeat(10) + + " */" + + const sql = `with ${longComment} "quoted_cte_name" as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"quoted_cte_name"', + 0, + "CTE with extremely long block comment (1500+ chars) before quoted name", + ) + }) + + it("should handle extremely long line comment before quoted CTE name", () => { + // Create a 1000+ character line comment + const longLineComment = + "-- " + + "This is an extremely long line comment that might appear in real-world SQL files. ".repeat(12) + + "It contains 'quoted strings' and special characters that could cause issues. " + + const sql = `with\n${longLineComment}\n"quoted_cte_name" as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"quoted_cte_name"', + 0, + "CTE with extremely long line comment (1000+ chars) before quoted name", + ) + }) + + it("should handle extremely long Jinja comment before quoted CTE name", () => { + // Create a 1200+ character Jinja comment + const longJinjaComment = + "{# " + + "This is an extremely long Jinja comment that might appear in real-world dbt files. ".repeat(15) + + "It contains 'quoted strings', template variables like {{ var('my_var') }}, and complex logic. " + + "The comment includes special characters and database keywords that could confuse parsers. " + + " #}" + + const sql = `with ${longJinjaComment} "quoted_cte_name" as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"quoted_cte_name"', + 0, + "CTE with extremely long Jinja comment (1200+ chars) before quoted name", + ) + }) + + it("should handle extremely long block comment after quoted CTE name", () => { + // Create a 2000+ character comment after the name + const longComment = + "/* " + + "This is an extremely long comment that appears after the table name but before AS. ".repeat(30) + + "It contains multiple paragraphs, 'quoted strings', and complex descriptions that might " + + "explain the business logic behind the transformation. The comment includes database keywords, special " + + "characters, and other potentially problematic content for regex parsing. ".repeat(5) + + " */" + + const sql = `with "quoted_cte_name" ${longComment} as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"quoted_cte_name"', + 0, + "CTE with extremely long block comment (2000+ chars) after quoted name", + ) + }) + + it("should handle multiple very long comments around quoted CTE name", () => { + // Create multiple long comments around the CTE name + const longCommentBefore = + "/* " + "Long comment before the CTE name. ".repeat(20) + "Contains 'quotes' and special characters. " + " */" + + const longCommentAfter = + "/* " + + "Long comment after the CTE name but before AS. ".repeat(20) + + "Also contains 'quotes' and special characters. " + + " */" + + const sql = `with ${longCommentBefore} "quoted_cte_name" ${longCommentAfter} as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], '"quoted_cte_name"', 0, "CTE with multiple very long comments around quoted name") + }) + + it("should handle extremely long mixed comment types around quoted CTE name", () => { + // Test with line comment, block comment, and Jinja comment (avoid SQL keywords in comments) + const longLineComment = "-- " + "Very long line comment containing 'quotes' and special characters. ".repeat(10) + + const longBlockComment = + "/* " + "Very long block comment containing 'quotes' and special characters. ".repeat(15) + " */" + + const longJinjaComment = + "{# " + "Very long Jinja comment containing 'quotes' and template logic. ".repeat(10) + " #}" + + const sql = `${longLineComment}\n${longBlockComment}\nwith "quoted_cte_name" ${longJinjaComment} as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], '"quoted_cte_name"', 0, "CTE with extremely long mixed comment types around quoted name") + }) + + it("should handle lengthy comments with quoted names in multi-CTE scenario", () => { + // Test performance with multiple CTEs each having lengthy comments + const longComment1 = + "/* " + + "First data source containing very long description. ".repeat(25) + + "Contains 'quotes' and special characters for testing. " + + " */" + + const longComment2 = + "/* " + + "Second data source containing very long description. ".repeat(25) + + "Also contains 'quotes' and special characters for testing. " + + " */" + + const sql = `with + ${longComment1} "first_quoted_cte" as ( + select 1 as id, 'first' as name + ), + ${longComment2} "second_quoted_cte" as ( + select 2 as id, 'second' as name + ) + select * from "first_quoted_cte" union all select * from "second_quoted_cte"` + + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(2) + assertCTE(ctes[0], '"first_quoted_cte"', 0, "First CTE with very long comment before quoted name") + assertCTE(ctes[1], '"second_quoted_cte"', 1, "Second CTE with very long comment before quoted name") + }) + + it("should handle extremely long comments with regex-problematic content", () => { + // Create a comment with content that could cause regex issues + const problemComment = + "/* " + + "This comment contains regex-problematic content: .*+?^${}()|[] ".repeat(20) + + "It also has nested quotes like 'text with \"nested quotes\" inside' and " + + "backslashes \\ and forward slashes // that could break poorly designed regex. " + + "The comment includes SQL injection attempts like '; DROP TABLE users; -- " + + "and other malicious content that should be handled safely. ".repeat(10) + + " */" + + const sql = `with ${problemComment} "secure_cte_name" as (select 1 as id) select * from "secure_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], '"secure_cte_name"', 0, "CTE with extremely long comment containing regex-problematic content") + }) + }) + + describe("Long quoted strings inside comments", () => { + it("should ignore a long quoted string inside a comment", () => { + // Create a quoted string in comment that's longer than 500 chars + const veryLongQuotedString = '"' + "a".repeat(600) + '"' + const commentWithLongQuotedString = `/* This comment contains a very long quoted string: ${veryLongQuotedString} that is long */` + + const sql = `with ${commentWithLongQuotedString} "real_cte" as (select 1 as id) select * from "real_cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"real_cte"', + 0, + "CTE with comment containing quoted string longer than 500 characters", + ) + }) + + it("should handle quoted CTE name approaching 500 characters", () => { + // Create a quoted CTE name that's close to but under the 500 char limit + const longCTEName = '"' + "very_long_cte_name_" + "x".repeat(450) + '"' + + const sql = `with ${longCTEName} as (select 1 as id) select * from ${longCTEName}` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], longCTEName, 0, "CTE with quoted name approaching 500 characters") + }) + + it("should handle quoted CTE name exactly at 500 characters", () => { + // Create a quoted CTE name that's exactly at the 500 char limit (498 chars inside quotes + 2 quotes = 500) + const exactLimitCTEName = '"' + "x".repeat(498) + '"' + + const sql = `with ${exactLimitCTEName} as (select 1 as id) select * from ${exactLimitCTEName}` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], exactLimitCTEName, 0, "CTE with quoted name exactly at 500 characters") + }) + + it("should handle comment with multiple long quoted strings and real CTE", () => { + // Test multiple long quoted strings in comments + const longQuotedString1 = '"' + "fake_identifier_1_" + "a".repeat(600) + '"' + const longQuotedString2 = '"' + "fake_identifier_2_" + "b".repeat(700) + '"' + const commentWithMultipleLongQuotes = `/* Comment with ${longQuotedString1} and also ${longQuotedString2} */` + + const sql = `with ${commentWithMultipleLongQuotes} "actual_cte" as (select 1 as id) select * from "actual_cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"actual_cte"', + 0, + "CTE with comment containing multiple quoted strings longer than 500 characters", + ) + }) + + it("should handle extremely long comment with quoted content and real long CTE name", () => { + // Combine very long comment with quoted content AND a long CTE name + const longQuotedInComment = '"' + "fake_cte_name_in_comment_" + "z".repeat(800) + '"' + const longComment = `/* ${"Very long comment text. ".repeat(50)} Contains fake quoted identifier: ${longQuotedInComment} ${"More comment text. ".repeat(30)} */` + const longRealCTEName = '"' + "real_cte_name_" + "r".repeat(400) + '"' + + const sql = `with ${longComment} ${longRealCTEName} as (select 1 as id) select * from ${longRealCTEName}` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + longRealCTEName, + 0, + "CTE with extremely long comment containing long quoted string and long real CTE name", + ) + }) + + it("should handle line comment with long quoted string before quoted CTE", () => { + // Test line comment with long quoted content + const longQuotedInLineComment = '"' + "fake_line_comment_identifier_" + "l".repeat(600) + '"' + const lineComment = `-- Line comment with long quoted string: ${longQuotedInLineComment}` + + const sql = `with\n${lineComment}\n"real_cte" as (select 1 as id) select * from "real_cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"real_cte"', + 0, + "CTE with line comment containing quoted string longer than 500 characters", + ) + }) + + it("should handle Jinja comment with long quoted string before quoted CTE", () => { + // Test Jinja comment with long quoted content + const longQuotedInJinjaComment = '"' + "fake_jinja_identifier_" + "j".repeat(650) + '"' + const jinjaComment = `{# Jinja comment with long quoted string: ${longQuotedInJinjaComment} #}` + + const sql = `with ${jinjaComment} "real_cte" as (select 1 as id) select * from "real_cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"real_cte"', + 0, + "CTE with Jinja comment containing quoted string longer than 500 characters", + ) + }) + + it("should handle mixed comment types with various length quoted strings", () => { + // Test combination of different comment types with various length quoted strings + const shortQuoted = '"short"' + const mediumQuoted = '"' + "medium_" + "m".repeat(200) + '"' + const longQuoted = '"' + "very_long_" + "v".repeat(800) + '"' + + const lineComment = `-- Line comment with ${shortQuoted}` + const blockComment = `/* Block comment with ${mediumQuoted} and ${longQuoted} */` + const jinjaComment = `{# Jinja comment with ${longQuoted} #}` + + const sql = `${lineComment}\n${blockComment}\nwith ${jinjaComment} "mixed_test_cte" as (select 1 as id) select * from "mixed_test_cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], '"mixed_test_cte"', 0, "CTE with mixed comment types containing various length quoted strings") + }) + }) + + describe("CTE Column List Test Cases", () => { + describe("Basic Column List Variations", () => { + it("should handle simple column list with multiple columns", () => { + const sql = + "with my_cte (id, name, email, created_at) as (select 1, 'John', 'john@example.com', now()) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (id, name, email, created_at)", 0, "CTE with multiple column names in list") + }) + + it("should handle column list with quoted column names", () => { + const sql = + 'with my_cte ("user id", "full name", "email address") as (select 1, \'John Doe\', \'john@example.com\') select * from my_cte' + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + 'my_cte ("user id", "full name", "email address")', + 0, + "CTE with quoted column names containing spaces", + ) + }) + + it("should handle column list with mixed quoted and unquoted names", () => { + const sql = + "with my_cte (id, \"user name\", email, \"created at\") as (select 1, 'John', 'john@example.com', now()) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + 'my_cte (id, "user name", email, "created at")', + 0, + "CTE with mixed quoted and unquoted column names", + ) + }) + + it("should handle column list with simple data types", () => { + const sql = + "with my_cte (id integer, name varchar, amount decimal) as (select 1, 'John', 100.50) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + "my_cte (id integer, name varchar, amount decimal)", + 0, + "CTE with simple column names and data types", + ) + }) + + it("should handle column list with single column", () => { + const sql = "with my_cte (single_column) as (select 'value') select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (single_column)", 0, "CTE with single column in list") + }) + }) + + describe("Length and Performance Edge Cases", () => { + it("should handle column list approaching MAX_COLUMN_LIST_LENGTH limit", () => { + // Create a column list that's close to but under the 1000 char limit + const longColumnList = Array.from({ length: 20 }, (_, i) => `column_${i}_with_long_name`).join(", ") + const sql = `with my_cte (${longColumnList}) as (select ${Array.from({ length: 20 }, (_, i) => i).join(", ")}) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte (${longColumnList})`, + 0, + "CTE with column list approaching MAX_COLUMN_LIST_LENGTH limit", + ) + }) + + it("should handle column list near MAX_COLUMN_LIST_LENGTH limit", () => { + // Create a column list that's close to the 1000 char limit + const nearLimitColumnList = "x".repeat(900) // 900 chars - well within limit + const sql = `with my_cte (${nearLimitColumnList}) as (select 1) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte (${nearLimitColumnList})`, + 0, + "CTE with column list near MAX_COLUMN_LIST_LENGTH limit", + ) + }) + + it("should handle very long individual column names", () => { + const longColumnName1 = '"' + "very_long_column_name_" + "a".repeat(200) + '"' + const longColumnName2 = '"' + "another_long_column_name_" + "b".repeat(200) + '"' + const sql = `with my_cte (${longColumnName1}, ${longColumnName2}) as (select 1, 2) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte (${longColumnName1}, ${longColumnName2})`, + 0, + "CTE with very long individual column names", + ) + }) + }) + + describe("Complex Column List Scenarios", () => { + it("should handle column list with nested parentheses (function calls)", () => { + const sql = + "with my_cte (id, calculated_value, formatted_date) as (select 1, round(avg(amount), 2), to_char(created_at, 'YYYY-MM-DD')) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + "my_cte (id, calculated_value, formatted_date)", + 0, + "CTE with column list containing nested parentheses from function calls", + ) + }) + + it("should handle column list with simple constraints", () => { + const sql = + "with my_cte (id not null, name, amount default 0) as (select 1, 'John', 100.0) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + "my_cte (id not null, name, amount default 0)", + 0, + "CTE with column list containing simple constraints", + ) + }) + + it("should handle column list with line breaks and formatting", () => { + const sql = `with my_cte ( + id, + name, + email, + created_at + ) as ( + select 1, 'John', 'john@example.com', now() + ) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte ( + id, + name, + email, + created_at + )`, + 0, + "CTE with multi-line formatted column list", + ) + }) + + it("should handle column list with special characters and escaped quotes", () => { + const sql = `with my_cte ("col_with_'quote", "col with spaces", "col@with#symbols") as (select 1, 2, 3) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte ("col_with_'quote", "col with spaces", "col@with#symbols")`, + 0, + "CTE with column list containing special characters and quotes", + ) + }) + }) + + describe("Column List + Comment Combinations", () => { + it("should handle comments before the opening parenthesis", () => { + const sql = "with my_cte /* comment before column list */ (id, name) as (select 1, 'John') select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (id, name)", 0) + }) + + it("should handle comments after the closing parenthesis", () => { + const sql = "with my_cte (id, name) /* comment after column list */ as (select 1, 'John') select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (id, name)", 0, "CTE with comment after column list closing parenthesis") + }) + + it("should handle comments between column list and AS keyword", () => { + const sql = `with my_cte (id, name, email) + -- This is a comment before AS + /* Another comment */ + as (select 1, 'John', 'john@example.com') select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (id, name, email)", 0, "CTE with comments between column list and AS keyword") + }) + + it("should handle mixed comment types around column lists", () => { + const sql = `with /* before */ my_cte ( + id, -- line comment + name /* inline comment */, + email + ) {# jinja comment #} as (select 1, 'John', 'john@example.com') select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + // Comments within column lists are currently included in the CTE name + // This is expected behavior given the current parsing approach + assertCTE( + ctes[0], + `my_cte ( + id, -- line comment + name /* inline comment */, + email + )`, + 0, + "CTE with mixed comment types around column list", + ) + }) + }) + + describe("Error Handling and Edge Cases", () => { + it("should handle empty column lists", () => { + const sql = "with my_cte () as (select 1) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte ()", 0, "CTE with empty column list") + }) + + it("should handle column lists with only whitespace", () => { + const sql = "with my_cte ( ) as (select 1) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte ( )", 0, "CTE with whitespace-only column list") + }) + + it("should handle column lists in combination with quoted CTE names", () => { + const sql = + 'with "my cte" (id, "user name", email) as (select 1, \'John\', \'john@example.com\') select * from "my cte"' + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"my cte" (id, "user name", email)', + 0, + "CTE with quoted name and column list with quoted columns", + ) + }) + + it("should handle deeply nested parentheses in column definitions", () => { + const sql = + "with my_cte (calculated_field, nested_calc) as (select sum(case when (x > 0 and (y < 10 or z = (a + b))) then 1 else 0 end), avg(coalesce(field1, field2, 0))) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + "my_cte (calculated_field, nested_calc)", + 0, + "CTE with column list and deeply nested parentheses in query", + ) + }) + }) + + describe("Real-World Complex Scenarios", () => { + it("should handle simple BigQuery-style column definitions", () => { + const sql = `with my_cte ( + user_id INT64, + user_name STRING, + created_at TIMESTAMP + ) as ( + select 1, 'John', current_timestamp() + ) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte ( + user_id INT64, + user_name STRING, + created_at TIMESTAMP + )`, + 0, + "CTE with simple BigQuery-style column definitions", + ) + }) + + it("should handle multi-CTE scenarios with varying column list complexity", () => { + const sql = `with + simple_cte (id, name) as ( + select 1, 'John' + ), + complex_cte ( + user_id, + "full name", + calculated_field, + metadata_json + ) as ( + select id, name, id * 100, '{"key": "value"}' + from simple_cte + ), + no_columns_cte as ( + select * from complex_cte + ) + select * from no_columns_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "simple_cte (id, name)", 0, "First CTE with simple column list") + assertCTE( + ctes[1], + `complex_cte ( + user_id, + "full name", + calculated_field, + metadata_json + )`, + 1, + "Second CTE with complex multi-line column list", + ) + assertCTE(ctes[2], "no_columns_cte", 2, "Third CTE without column list") + }) + }) + }) + + describe("CTE Cross-References", () => { + it("should handle simple sequential CTE references", () => { + const sql = `with + base_data as ( + select 1 as id, 'test' as name + ), + filtered_data as ( + select * from base_data where id > 0 + ), + final_data as ( + select id, upper(name) as name from filtered_data + ) + select * from final_data` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "base_data", 0, "CTE with hardcoded test data") + assertCTE(ctes[1], "filtered_data", 1, "CTE filtering data from base_data CTE") + assertCTE(ctes[2], "final_data", 2, "CTE transforming data from filtered_data CTE") + }) + + it("should handle CTE references with joins", () => { + const sql = `with + orders as ( + select 1 as order_id, 100 as customer_id + ), + customers as ( + select 100 as id, 'John' as name + ), + order_details as ( + select o.order_id, c.name + from orders o + join customers c on o.customer_id = c.id + ) + select * from order_details` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "orders", 0, "CTE with order and customer ID data") + assertCTE(ctes[1], "customers", 1, "CTE with customer ID and name data") + assertCTE(ctes[2], "order_details", 2, "CTE joining orders and customers CTEs") + }) + + it("should handle CTE references with comments and quotes", () => { + const sql = `with + source_data as ( + select id, name -- 'source' data extraction + from raw_table + ), + cleaned_data as ( + /* Reference to 'source_data' CTE */ + select * from source_data + where name is not null + ) + select * from cleaned_data` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(2) + assertCTE(ctes[0], "source_data", 0, "CTE with line comment containing single quotes") + assertCTE(ctes[1], "cleaned_data", 1, "CTE referencing source_data with block comment containing quotes") + }) + + it("should handle multiple CTE references in one query", () => { + const sql = `with + cte_a as ( + select 1 as id, 'A' as source + ), + cte_b as ( + select 2 as id, 'B' as source + ), + cte_combined as ( + select * from cte_a + union all + select * from cte_b + ) + select * from cte_combined` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "cte_a", 0, "CTE with ID 1 and source A") + assertCTE(ctes[1], "cte_b", 1, "CTE with ID 2 and source B") + assertCTE(ctes[2], "cte_combined", 2, "CTE combining cte_a and cte_b with UNION ALL") + }) + + it("should handle nested CTE references with subqueries", () => { + const sql = `with + base as ( + select id, amount from transactions + ), + aggregated as ( + select id, sum(amount) as total + from base + group by id + ), + ranked as ( + select *, + (select count(*) from aggregated a2 where a2.total > aggregated.total) as rank + from aggregated + ) + select * from ranked` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "base", 0, "CTE selecting ID and amount from transactions table") + assertCTE(ctes[1], "aggregated", 1, "CTE grouping base CTE data by ID with sum") + assertCTE(ctes[2], "ranked", 2, "CTE adding rank using correlated subquery on aggregated CTE") + }) + }) + + describe("Edge Cases", () => { + it("should handle no CTEs", () => { + const sql = "select * from my_table" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(0) + }) + + it("should handle empty WITH clause", () => { + const sql = "with select * from my_table" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(0) + }) + + it("should handle complex nested queries", () => { + const sql = `with my_cte as ( + select id, ( + select count(*) + from other_table + where other_table.id = main_table.id + ) as count + from main_table + ) select * from my_cte` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with correlated subquery counting from other table") + + }) + + it("should handle unterminated comments gracefully", () => { + const sql = "with source /* unterminated comment as (select 1) select * from source" + const document = sql + + const ctes = detectCtes(document) + + // The comment swallows the rest of the text, so there is no CTE to find. + expect(ctes).toEqual([]) + }) + }) + + describe("CTE Range and Content Validation", () => { + it("should provide correct ranges for CTE name and query", () => { + const sql = "with my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + const cte = ctes[0] + + // Verify basic range structure is defined + expect(typeof cte.nameStart).toBe("number") + expect(typeof cte.queryStart).toBe("number") + expect(cte.nameStart).toBeDefined() + expect(cte.nameEnd).toBeDefined() + expect(cte.queryStart).toBeDefined() + expect(cte.queryEnd).toBeDefined() + + // Verify the CTE name was extracted correctly + expect(cte.name).toBe("my_cte") + expect(cte.index).toBe(0) + expect(typeof cte.withClauseStart).toBe("number") + }) + + it("should detect complex CTEs correctly", () => { + const sql = `with + orders_summary as ( + select + customer_id, + count(*) as order_count, + sum(total_amount) as total_spent + from orders + where status = 'completed' + group by customer_id + ) + select * from orders_summary` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "orders_summary", 0, "CTE with aggregated customer order data") + + // Verify range structure is correct + expect(typeof ctes[0].queryStart).toBe("number") + }) + + it("should handle CTE detection with comments correctly", () => { + const sql = `with my_cte as ( + select + id, + name, -- This is a comment with quotes 'inside' + /* Another comment */ + status + from users + {# Jinja comment with 'quotes' #} + where active = true + ) select * from my_cte` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with line, block, and Jinja comments containing quotes") + + // Verify that CTE was detected despite comments with quotes + expect(ctes[0].name).toBe("my_cte") + expect(typeof ctes[0].queryStart).toBe("number") + }) + }) +}) + +describe("detectCtes offsets", () => { + it("returns offsets that slice the name and the body", () => { + const sql = "with my_cte as (select 1 as id) select * from my_cte" + const [cte] = detectCtesInText(sql) + expect(sql.slice(cte.nameStart, cte.nameEnd)).toBe("my_cte") + expect(sql.slice(cte.queryStart, cte.queryEnd)).toBe("select 1 as id") + expect(cte.withClauseStart).toBe(0) + }) + + it("keeps the column list in the name", () => { + const sql = "with my_cte (id, name) as (select 1, 'test') select * from my_cte" + const [cte] = detectCtesInText(sql) + expect(cte.name).toBe("my_cte (id, name)") + expect(stripColumnList(cte.name)).toBe("my_cte") + }) + + it("does not detect a WITH inside a comment or string", () => { + const sql = "-- with fake as (select 1)\nselect 'with x as (select 2)' as s from t" + expect(detectCtesInText(sql)).toEqual([]) + }) + + it("lists the CTEs of each top-level WITH, each with its own clause", () => { + const sql = "with a as (select 1) with b as (select 2) select * from b" + const ctes = detectCtesInText(sql) + expect(ctes.map((c) => [c.name, c.index])).toEqual([["a", 0], ["b", 0]]) + expect(ctes[0].withClauseStart).not.toBe(ctes[1].withClauseStart) + }) + + it("lists only the model's own CTEs: a WITH inside a CTE body is part of that body", () => { + const sql = `with outer_cte as ( + with inner_cte as (select 1 as x) + select x from inner_cte +), +last as (select * from outer_cte) +select * from last` + const ctes = detectCtesInText(sql) + expect(ctes.map((c) => [c.name, c.index])).toEqual([["outer_cte", 0], ["last", 1]]) + // The outer CTE's count query carries its nested WITH whole and closes every parenthesis. + const query = buildCountQuery(sql, ctes, ctes[1])! + expect(query).toContain("inner_cte as (select 1 as x)") + expect(query.match(/\(/g)!.length).toBe(query.match(/\)/g)!.length) + expect(query).toEndWith("SELECT COUNT(*) AS _profile_count FROM last") + }) + + it("a WITH inside a subquery or a window definition is not a model CTE", () => { + const sql = "select * from (with s as (select 1) select * from s) t join u on true window w as (partition by a)" + expect(detectCtesInText(sql)).toEqual([]) + }) + + it("comment markers and CTE-like text inside string literals neither hide nor add CTEs", () => { + expect(detectCtesInText("with a as (select '--' as s), b as (select 2) select * from b").map((c) => c.name)).toEqual(["a", "b"]) + expect(detectCtesInText("with a as (select '/*' as s), b as (select '*/' as t) select * from b").map((c) => c.name)).toEqual(["a", "b"]) + expect(detectCtesInText("with a as (select '(x)' as s, 'fake as (select 2)' as t) select * from a").map((c) => c.name)).toEqual(["a"]) + expect(detectCtesInText("with a as (select 'it''s ) as (' as s), b as (select 2) select * from b").map((c) => c.name)).toEqual(["a", "b"]) + }) + + it("a quoted name keeps its escaped quote", () => { + const sql = 'with "a""b" as (select 1), `c``d` as (select 2) select * from "a""b"' + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(['"a""b"', "`c``d`"]) + }) + + it("skips a captured {% set %} block between CTEs and reads a bracket name with an escaped ]]", () => { + const sql = "with a as (select 1),\n{% set helper %}select 2{% endset %}\nb as (select 2)\nselect * from b" + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["a", "b"]) + expect(detectCtesInText("with [we]]ird] as (select 1) select * from [we]]ird]").map((c) => c.name)).toEqual(["[we]]ird]"]) + }) + + it("keeps CTEs that follow a Jinja tag between them, and reads WITH RECURSIVE and MATERIALIZED", () => { + const jinja = "with a as (select 1),\n{% if x %}\nb as (select 2),\n{% endif %}\nc as (select 3)\nselect * from c" + expect(detectCtesInText(jinja).map((c) => c.name)).toEqual(["a", "b", "c"]) + expect(detectCtesInText("with recursive r as (select 1) select * from r").map((c) => c.name)).toEqual(["r"]) + expect(detectCtesInText("with m as materialized (select 1), n as not materialized (select 2) select 1").map((c) => c.name)).toEqual(["m", "n"]) + }) + + it("scans in linear time: a 2 MB model is read in well under a second", () => { + const body = "select a, b, c from t where x = 1 and y = 'it''s' -- note\n" + const sql = `with big as (\n${body.repeat(Math.floor(2_000_000 / body.length))}) select * from big` + const started = performance.now() + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["big"]) + expect(performance.now() - started).toBeLessThan(1000) + }) + + it("keeps WITH RECURSIVE in the count query, and a WITH written inside a Jinja block is not the model's", () => { + const recursive = "with recursive r as (select 1 as n union all select n + 1 from r where n < 3) select * from r" + const ctes = detectCtesInText(recursive) + expect(buildCountQuery(recursive, ctes, ctes[0])).toStartWith("with recursive r as (") + const block = "{% set q %}with z as (select 1) select * from z{% endset %}\nwith a as (select 1), b as (select 2) select * from b" + const found = detectCtesInText(block) + expect(found.map((c) => c.name)).toEqual(["a", "b"]) + expect(buildCountQuery(block, found, found[1])).toContain("b as (select 2)") + const inline = "{% set q = 'with z as (select 1)' %}\nwith a as (select 1) select * from a" + expect(detectCtesInText(inline).map((c) => c.name)).toEqual(["a"]) + const macro = "{% macro m() %}with z as (select 1) select * from z{% endmacro %}\nwith a as (select 1) select * from a" + expect(detectCtesInText(macro).map((c) => c.name)).toEqual(["a"]) + }) + + it("reads $ and non-ASCII letters in a name, and survives crafted input in linear time", () => { + expect(detectCtesInText("with my$cte as (select 1), é_t as (select 2) select * from é_t").map((c) => c.name)).toEqual(["my$cte", "é_t"]) + for (const unit of ["with [", "with {%", "with ((", "with a as (", "with '", "with /*", "with \"x"]) { + const started = performance.now() + detectCtesInText(unit.repeat(200_000)) + expect(performance.now() - started).toBeLessThan(2000) + } + }) +}) + +describe("buildCountQuery", () => { + const sql = [ + "{{ config(materialized='table') }}", + "", + "with orders as (select * from {{ ref('stg_orders') }}),", + "customers as (select * from {{ ref('stg_customers') }}),", + "joined as (select o.id from orders o join customers c on o.cid = c.id)", + "select * from joined", + ].join("\n") + + it("keeps the preamble and every earlier CTE up to the target", () => { + const ctes = detectCtesInText(sql) + expect(ctes.map((c) => c.name)).toEqual(["orders", "customers", "joined"]) + const query = buildCountQuery(sql, ctes, ctes[1])! + expect(query.startsWith("{{ config(materialized='table') }}")).toBe(true) + expect(query).toContain("with orders as (") + expect(query).toContain("customers as (") + expect(query).not.toContain("joined") + expect(query.endsWith("SELECT COUNT(*) AS _profile_count FROM customers")).toBe(true) + }) + + it("counts the last CTE with all of its inputs", () => { + const ctes = detectCtesInText(sql) + const query = buildCountQuery(sql, ctes, ctes[2])! + expect(query).toContain("joined as (") + expect(query).toContain("from orders o join customers c") + expect(query.endsWith("FROM joined")).toBe(true) + }) + + it("keeps a column list in the definition but not in the FROM clause", () => { + const text = "with a (x, y) as (select 1, 2) select * from a" + const ctes = detectCtesInText(text) + const query = buildCountQuery(text, ctes, ctes[0])! + expect(query).toContain("with a (x, y) as (") + expect(query.endsWith("FROM a")).toBe(true) + }) + + it("builds no query for a CTE of a second WITH clause, whose preamble would hold the first clause's query", () => { + const text = "with a as (select 1) select * from a;\nwith b as (select 2) select * from b" + const ctes = detectCtesInText(text) + expect(buildCountQuery(text, ctes, ctes.find((c) => c.name === "a")!)).toContain("with a as (") + expect(buildCountQuery(text, ctes, ctes.find((c) => c.name === "b")!)).toBeUndefined() + }) + + it("returns undefined for a CTE that is not in the list", () => { + expect( + buildCountQuery("select 1", [], { + name: "x", + nameStart: 0, + nameEnd: 1, + queryStart: 0, + queryEnd: 0, + index: 0, + withClauseStart: 0, + }), + ).toBeUndefined() + }) +}) + +describe("count query edge cases", () => { + it("keeps Jinja written between CTEs, and reports a block left open by the cut", () => { + const kept = "{% set n = 2 %}\nwith a as (select {{ n }}),\n{% if true %}b as (select 2),{% endif %}\nc as (select 3) select * from c" + const ctes = detectCtesInText(kept) + expect(buildCountQuery(kept, ctes, ctes[2])).toContain("{% if true %}b as (select 2),{% endif %}") + expect(unclosedJinjaBlock(buildCountQuery(kept, ctes, ctes[2])!)).toBeUndefined() + const open = "{% if x %}\nwith a as (select 1) select * from a\n{% endif %}" + const found = detectCtesInText(open) + expect(unclosedJinjaBlock(buildCountQuery(open, found, found[0])!)).toBe("if") + expect(unclosedJinjaBlock("{% set q %}x")).toBe("set") + expect(unclosedJinjaBlock("{% raw %}{% if x %}{% endraw %}")).toBeUndefined() + expect(unclosedJinjaBlock("{% block body %}with a as (select 1)")).toBe("block") + expect(unclosedJinjaBlock("{% if x %}a{%+ endif %}")).toBeUndefined() + expect(unclosedJinjaBlock("{% set (a, b) = (1, 2) %}x")).toBeUndefined() + }) + + it("matches nested set blocks to their own end tag", () => { + const sql = "{% set outer %}{% set inner %}with fake as (select 1){% endset %}x{% endset %}\nwith real as (select 2) select * from real" + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["real"]) + }) + + it("reads a comment or Jinja tag between AS, NOT and MATERIALIZED and the body", () => { + expect(detectCtesInText("with a as /* c */ (select 1), b as not /* c */ materialized (select 2) select 1").map((c) => c.name)).toEqual(["a", "b"]) + expect(detectCtesInText("with a /* c */ as {# j #} materialized -- x\n(select 1) select 1").map((c) => c.name)).toEqual(["a"]) + }) + + it("strips a column list that holds a quoted ')' or a comment", () => { + expect(stripColumnList('a ("x)")')).toBe("a") + expect(stripColumnList("a (x /* ) */)")).toBe("a") + expect(stripColumnList("Big Spenders")).toBe("Big Spenders") + const text = 'with a ("x)", y) as (select 1, 2) select * from a' + const ctes = detectCtesInText(text) + expect(buildCountQuery(text, ctes, ctes[0])).toEndWith("FROM a") + }) + + it("a dollar-quoted string does not end the CTE body early", () => { + const text = "with a as (select $$ ) $$ as s), b as (select $tag$ ( $tag$ as t) select 1" + expect(detectCtesInText(text).map((c) => c.name)).toEqual(["a", "b"]) + }) + + it("tags inside a Jinja comment are not blocks, and a % inside a tag does not hide it", () => { + expect(unclosedJinjaBlock("{# {% if x %} #}with a as (select 1)")).toBeUndefined() + expect(unclosedJinjaBlock("{% if x %}{# {% endif %} #}with a as (select 1)")).toBe("if") + expect(unclosedJinjaBlock("{% if x % 2 == 0 %}with a as (select 1)")).toBe("if") + const sql = "{% set q %}{# {% endset %} #}with z as (select 1){% endset %}\nwith a as (select 2) select * from a" + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["a"]) + }) + + it("counts open Jinja blocks in linear time: 100k tags in a second", () => { + const sql = "{% if x %}".repeat(100_000) + "{% endfor %}".repeat(100_000) + const started = performance.now() + expect(unclosedJinjaBlock(sql)).toBe("if") + expect(performance.now() - started).toBeLessThan(1000) + }) + + it("unterminated tags, comments and many block tags are scanned in linear time", () => { + for (const unit of ["{% if x ", "{# c ", "{% set q %}", "{% set q | upper %}"]) { + const sql = unit.repeat(100_000) + "with a as (select 1) select 1" + const started = performance.now() + unclosedJinjaBlock(sql) + detectCtesInText(sql) + expect(performance.now() - started).toBeLessThan(1500) + } + }) + + it("a filtered set block is a block, and raw text holds no tags", () => { + expect(unclosedJinjaBlock("{% set q | upper %}x")).toBe("set") + expect(unclosedJinjaBlock("{% set q = 1 | upper %}x")).toBeUndefined() + const sql = "{% set q %}{% raw %}{% endset %}{% endraw %}with z as (select 1){% endset %}\nwith a as (select 2) select * from a" + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["a"]) + }) + + it("a dollar-quote tag may hold digits", () => { + expect(detectCtesInText("with a as (select $f1$ ) $f1$ as s), b as (select 2) select 1").map((c) => c.name)).toEqual(["a", "b"]) + }) + + it("writes a name with $ or a non-ASCII letter as it was declared", () => { + for (const name of ["my$cte", "Ünits"]) { + const text = `with ${name} as (select 1) select * from ${name}` + const ctes = detectCtesInText(text) + expect(buildCountQuery(text, ctes, ctes[0])).toEndWith(`FROM ${name}`) + } + }) +}) + +describe("quoteSqlIdentifier", () => { + it("leaves plain, quoted and dotted names alone and quotes the rest", () => { + expect(quoteSqlIdentifier("my_cte")).toBe("my_cte") + expect(quoteSqlIdentifier('"My Cte"')).toBe('"My Cte"') + expect(quoteSqlIdentifier("`x`")).toBe("`x`") + expect(quoteSqlIdentifier("[x]")).toBe("[x]") + expect(quoteSqlIdentifier("a.b")).toBe("a.b") + expect(quoteSqlIdentifier("my-cte")).toBe('"my-cte"') + }) + + it("is applied to a quoted CTE name in the count query", () => { + const text = 'with "My Cte" as (select 1) select * from "My Cte"' + const ctes = detectCtesInText(text) + expect(buildCountQuery(text, ctes, ctes[0])!.endsWith('FROM "My Cte"')).toBe(true) + }) +}) + +describe("extractRowCount", () => { + it("reads the count by position for upper- and lower-case aliases", () => { + expect(extractRowCount([{ _PROFILE_COUNT: 99 }])).toBe(99) + expect(extractRowCount([{ _profile_count: 42 }])).toBe(42) + }) + + it("coerces string counts", () => { + expect(extractRowCount([{ _PROFILE_COUNT: "1234" }])).toBe(1234) + }) + + it("returns 0 for no rows, a missing value or a non-numeric value", () => { + expect(extractRowCount([])).toBe(0) + expect(extractRowCount(undefined)).toBe(0) + expect(extractRowCount([{ _PROFILE_COUNT: "not-a-number" }])).toBe(0) + }) +}) diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts new file mode 100644 index 0000000000..fdafe2241f --- /dev/null +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -0,0 +1,271 @@ +// The tools the dbt Power User features call: their allow-list entries and placeholder answers, the +// one gate every dbt-adapter-backed tool goes through, and the API helper for new endpoint files. The +// dbt adapter is replaced through `dbtAdapterInternals`, so these tests need no Python or dbt. +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { realpathSync } from "node:fs" +import os from "node:os" +import path from "node:path" +import { Effect } from "effect" +import type { Tool } from "@/tool/tool" +import { SessionID, MessageID } from "@/session/schema" +import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" +import { dbtAdapterAccessInternals, dbtAdapterFor } from "@/altimate/tools/dbt-adapter-access" +import { EXPOSED, MCP_OPEN_WRITE_TOOLS, WRITE_TOOLS } from "@/altimate/tool-cli/allow-list" +import { AltimateApi } from "@/altimate/api/client" +import { Instance } from "@/project/instance" +import { disposeInstance } from "@/effect/instance-registry" +import { initTool } from "./tool-fixture" + +/** Each tool with the smallest arguments its parameters accept. */ +const MINIMAL_ARGS: Record> = { + dbt_run_results: {}, + dbt_defer_state: { project_dir: "." }, + dbt_map_relations: { sql: "select 1" }, + docs_share_comments: { action: "list" }, + dbt_cost_estimate: {}, + dbt_health: {}, + dbt_sql_graph: { model: "customers" }, + dbt_docs_coverage: { model: "customers" }, + dbt_cte_profile: { model: "customers" }, +} +const FEATURE_TOOLS = Object.keys(MINIMAL_ARGS) + +const ctx: Tool.Context = { + sessionID: SessionID.make("ses_dbt_adapter_access"), + messageID: MessageID.make("msg_dbt_adapter_access"), + agent: "analyst", + abort: new AbortController().signal, + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, +} + +let project: string + +function useConfig() { + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: project, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) +} + +beforeAll(async () => { + project = await mkdtemp(path.join(os.tmpdir(), "dbt-adapter-access-")) + await writeFile(path.join(project, "dbt_project.yml"), "name: jaffle_shop\n") +}) + +afterAll(async () => { + await rm(project, { recursive: true, force: true }) +}) + +beforeEach(() => resetDbtAdapter()) + +afterEach(() => { + resetDbtAdapter() + delete dbtAdapterInternals.readConfig + delete dbtAdapterInternals.create +}) + +describe("feature tools in the allow-list", () => { + test("all nine are exposed, in the map's alphabetical order, and none is a write tool", () => { + const ids = Object.keys(EXPOSED) + for (const id of FEATURE_TOOLS) { + expect(ids).toContain(id) + expect(WRITE_TOOLS.has(id)).toBe(false) + expect(MCP_OPEN_WRITE_TOOLS.has(id)).toBe(false) + } + const listed = ids.filter((id) => id.startsWith("dbt_") || id.startsWith("docs_")) + expect(listed).toEqual([...listed].sort()) + }) + + test("each placeholder answers that it isn't implemented, as a failure", async () => { + for (const id of FEATURE_TOOLS) { + // Implemented tools have their own test file. + if (id === "dbt_cte_profile") continue + const tool = await initTool(EXPOSED[id] as any) + expect(tool.id).toBe(id) + const result = await tool.execute(MINIMAL_ARGS[id], ctx) + expect(result.title).toBe(`${id}: not implemented`) + expect(result.metadata).toEqual({ success: false, error: "not implemented", truncated: false }) + expect(result.output).toContain(id) + } + }) +}) + +describe("dbtAdapterFor", () => { + let created: number + beforeEach(() => { + created = 0 + useConfig() + dbtAdapterInternals.create = async () => { + created += 1 + return { isInitialized: () => true, dispose: async () => {} } + } + }) + + test("refuses a read-only call and never creates the adapter", async () => { + const access = await dbtAdapterFor({ ...ctx, extra: { forceReadOnly: true } }) + expect(access.adapter).toBeUndefined() + expect(access.refusal).toBe("read_only") + expect(access.reason).toContain("--allow-write") + expect(created).toBe(0) + }) + + test("refuses a plan-mode call", async () => { + const access = await dbtAdapterFor({ ...ctx, agent: "plan" }) + expect(access.refusal).toBe("read_only") + expect(created).toBe(0) + }) + + test("returns the adapter where writes are allowed", async () => { + const access = await dbtAdapterFor(ctx) + expect(access.refusal).toBeUndefined() + expect(access.adapter?.isInitialized()).toBe(true) + expect(created).toBe(1) + }) + + test("says when the adapter didn't load the project", async () => { + dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_loaded", reason: "the dbt adapter didn't load the project" }) + }) + + test("says there is no dbt project when the config finds none", async () => { + dbtAdapterInternals.readConfig = async () => null + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "no_project", reason: "no dbt project here" }) + }) + + test("says when the adapter doesn't start in time", async () => { + dbtAdapterInternals.create = () => new Promise(() => {}) + const saved = dbtAdapterAccessInternals.adapterStartTimeoutMs + dbtAdapterAccessInternals.adapterStartTimeoutMs = 20 + try { + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start within 0.02 s" }) + } finally { + dbtAdapterAccessInternals.adapterStartTimeoutMs = saved + } + }) + + test("says the adapter didn't start when the project exists but creating it fails", async () => { + dbtAdapterInternals.create = async () => { + created += 1 + throw new Error("dbt-integration missing") + } + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start" }) + expect(created).toBe(1) + }) + + test("a call cancelled while the adapter starts throws and stops waiting", async () => { + let entered = () => {} + const starting = new Promise((resolve) => (entered = resolve)) + dbtAdapterInternals.create = () => { + entered() + return new Promise(() => {}) + } + const controller = new AbortController() + const pending = dbtAdapterFor({ ...ctx, abort: controller.signal }) + await starting + controller.abort(new Error("cancelled")) + await expect(pending).rejects.toThrow("cancelled") + }) + + test("a cancelled call throws rather than answering", async () => { + const controller = new AbortController() + controller.abort() + await expect(dbtAdapterFor({ ...ctx, abort: controller.signal })).rejects.toThrow() + expect(created).toBe(0) + }) + + test("a session in a dbt project the adapter isn't set up for says so", async () => { + dbtAdapterInternals.readConfig = async () => null + try { + const access = await Instance.provide({ directory: project, fn: () => dbtAdapterFor(ctx) }) + expect(access).toEqual({ + refusal: "other_project", + reason: `the dbt adapter isn't set up for ${realpathSync(project)}`, + }) + expect(created).toBe(0) + } finally { + await disposeInstance(project) + } + }) + + test("a session in another dbt project than the adapter's says which", async () => { + const other = await mkdtemp(path.join(os.tmpdir(), "dbt-adapter-other-")) + await writeFile(path.join(other, "dbt_project.yml"), "name: other\n") + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: other, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) + try { + const access = await Instance.provide({ directory: project, fn: () => dbtAdapterFor(ctx) }) + expect(access).toEqual({ + refusal: "other_project", + reason: `the dbt adapter is set up for ${realpathSync(other)}, not ${realpathSync(project)}`, + }) + expect(created).toBe(0) + } finally { + await disposeInstance(project) + await rm(other, { recursive: true, force: true }) + } + }) +}) + +describe("AltimateApi.apiRequest", () => { + let home: string + let saved: string | undefined + beforeEach(async () => { + home = await mkdtemp(path.join(os.tmpdir(), "altimate-api-home-")) + saved = process.env.OPENCODE_TEST_HOME + process.env.OPENCODE_TEST_HOME = home + }) + afterEach(async () => { + if (saved === undefined) delete process.env.OPENCODE_TEST_HOME + else process.env.OPENCODE_TEST_HOME = saved + await rm(home, { recursive: true, force: true }) + }) + + test("fails with a clear message when no credentials are stored", async () => { + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow( + `No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${path.join(home, ".altimate", "altimate.json")}).`, + ) + }) + + test("sends the stored key and tenant and returns the JSON body", async () => { + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: "https://api.example.test/", altimateInstanceName: "acme", altimateApiKey: "k1" }), + ) + const original = globalThis.fetch + const seen: { url: string; init: RequestInit }[] = [] + globalThis.fetch = (async (url: string, init: RequestInit) => { + seen.push({ url, init }) + return new Response(JSON.stringify({ ok: 1 }), { status: 200 }) + }) as unknown as typeof fetch + try { + expect(await AltimateApi.apiRequest("POST", "/dbt/v1/x", { a: 1 })).toEqual({ ok: 1 }) + expect(seen[0].url).toBe("https://api.example.test/dbt/v1/x") + const headers = seen[0].init.headers as Record + expect(headers.Authorization).toBe("Bearer k1") + expect(headers["x-tenant"]).toBe("acme") + expect(seen[0].init.body).toBe(JSON.stringify({ a: 1 })) + } finally { + globalThis.fetch = original + } + }) + + test("names the file when the stored credentials can't be read", async () => { + await Bun.write(path.join(home, ".altimate", "altimate.json"), "{not json") + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow( + `The Altimate credentials in ${path.join(home, ".altimate", "altimate.json")} can't be used`, + ) + }) +}) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts new file mode 100644 index 0000000000..0774adafd4 --- /dev/null +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -0,0 +1,400 @@ +// dbt_cte_profile: the CTE list, and one timed count query per CTE. The dbt adapter is replaced through +// `dbtAdapterInternals`, so no warehouse, Python or dbt is needed. +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { Effect } from "effect" +import type { Tool } from "@/tool/tool" +import { SessionID, MessageID } from "@/session/schema" +import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" +import { DbtCteProfileTool } from "@/altimate/tools/dbt-cte-profile" +import { initTool } from "./tool-fixture" + +const ctx: Tool.Context = { + sessionID: SessionID.make("ses_dbt_cte_profile"), + messageID: MessageID.make("msg_dbt_cte_profile"), + agent: "analyst", + abort: new AbortController().signal, + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, +} + +const CUSTOMERS = `{{ config(materialized='table') }} + +-- orders first +with orders as ( + select * from {{ ref('stg_orders') }} +), + +"Big Spenders" (customer_id, total) as ( + select customer_id, sum(amount) from orders group by 1 +), + +final as ( + select * from "Big Spenders" +) + +select * from final +` + +let project: string +let queries: { sql: string; name: string; limit: number }[] +let respond: (sql: string) => Promise + +function useAdapter() { + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: project, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) + dbtAdapterInternals.create = async () => ({ + isInitialized: () => true, + dispose: async () => {}, + immediatelyExecuteSQLWithLimit: async (sql: string, name: string, limit: number) => { + queries.push({ sql, name, limit }) + return respond(sql) + }, + }) +} + +beforeAll(async () => { + project = await mkdtemp(path.join(os.tmpdir(), "dbt-cte-profile-")) + await writeFile(path.join(project, "dbt_project.yml"), "name: jaffle_shop\n") + await mkdir(path.join(project, "models", "marts"), { recursive: true }) + await mkdir(path.join(project, "models", "staging"), { recursive: true }) + await mkdir(path.join(project, "dbt_packages", "pkg", "models"), { recursive: true }) + await writeFile(path.join(project, "models", "marts", "customers.sql"), CUSTOMERS) + await writeFile(path.join(project, "models", "staging", "plain.sql"), "select 1 as id\n") + await writeFile(path.join(project, "models", "marts", "dup.sql"), "with a as (select 1) select * from a") + await writeFile(path.join(project, "models", "staging", "dup.sql"), "with b as (select 1) select * from b") + await writeFile( + path.join(project, "models", "marts", "twins.sql"), + 'with "Foo" as (select 1 as a), foo as (select 2 as a), keep as (select 3 as a)\nselect * from keep\n', + ) + await writeFile( + path.join(project, "models", "marts", "wrapped.sql"), + "{% if is_incremental() %}\nwith a as (select 1 as x) select * from a\n{% else %}\nwith b as (select 2 as x) select * from b\n{% endif %}\n", + ) + await writeFile( + path.join(project, "models", "marts", "second.sql"), + "with a as (select 1 as x) select * from a;\nwith b as (select 2 as x) select * from b\n", + ) + await mkdir(path.join(project, "target"), { recursive: true }) + await writeFile( + path.join(project, "target", "manifest.json"), + JSON.stringify({ + nodes: { + "model.jaffle_shop.customers": { resource_type: "model", name: "customers", original_file_path: "models/marts/customers.sql" }, + "model.jaffle_shop.gone": { resource_type: "model", name: "gone", original_file_path: "models/marts/gone.sql" }, + "model.jaffle_shop.dup.v1": { resource_type: "model", name: "sales", unique_id: "model.jaffle_shop.sales.v1", original_file_path: "models/marts/dup.sql" }, + "model.jaffle_shop.sales.v2": { resource_type: "model", name: "sales", unique_id: "model.jaffle_shop.sales.v2", original_file_path: "models/staging/dup.sql" }, + "model.pkg.customers": { resource_type: "model", name: "customers", package_name: "pkg", original_file_path: "models/marts/customers.sql" }, + }, + }), + ) + await writeFile(path.join(project, "dbt_packages", "pkg", "models", "ignored.sql"), "with z as (select 1) select * from z") +}) + +afterAll(async () => { + await rm(project, { recursive: true, force: true }) +}) + +beforeEach(() => { + queries = [] + respond = async () => ({ columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }] }) + resetDbtAdapter() + useAdapter() +}) + +afterEach(() => { + resetDbtAdapter() + delete dbtAdapterInternals.readConfig + delete dbtAdapterInternals.create +}) + +describe("dbt_cte_profile list", () => { + test("lists the CTEs in source order with their lines, without the adapter, on a read-only call", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers" }, { ...ctx, extra: { forceReadOnly: true } }) + expect(result.metadata.success).toBe(true) + expect(result.metadata.file).toBe(path.join("models", "marts", "customers.sql")) + expect(result.metadata.ctes).toEqual([ + { name: "orders", line: 4, index: 0, clause: expect.any(Number), position: 0 }, + { name: '"Big Spenders"', line: 8, index: 1, clause: expect.any(Number), position: 1 }, + { name: "final", line: 12, index: 2, clause: expect.any(Number), position: 2 }, + ]) + expect(result.output).toContain("3 CTEs") + expect(queries).toEqual([]) + }) + + test("a model without CTEs lists none", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "plain" }, ctx) + expect(result.metadata.success).toBe(true) + expect(result.metadata.ctes).toEqual([]) + }) + + test("takes a project-relative path and ignores dbt_packages", async () => { + const tool = await initTool(DbtCteProfileTool) + const byPath = await tool.execute({ model: "models/marts/customers.sql" }, ctx) + expect(byPath.metadata.success).toBe(true) + const packaged = await tool.execute({ model: "ignored" }, ctx) + expect(packaged.metadata.success).toBe(false) + expect(packaged.output).toContain("no model named ignored") + }) + + test("an ambiguous name asks for the path; a path outside the project is refused", async () => { + const tool = await initTool(DbtCteProfileTool) + const dup = await tool.execute({ model: "dup" }, ctx) + expect(dup.metadata.success).toBe(false) + expect(dup.output).toContain("matches 2 files") + const outside = await tool.execute({ model: "../../etc/passwd.sql" }, ctx) + expect(outside.metadata.success).toBe(false) + expect(outside.output).toContain("outside the dbt project") + }) +}) + +describe("dbt_cte_profile model lookup", () => { + test("finds a model by unique_id through the manifest, and takes an absolute path", async () => { + const tool = await initTool(DbtCteProfileTool) + const byId = await tool.execute({ model: "model.jaffle_shop.customers" }, ctx) + expect(byId.metadata.success).toBe(true) + const absolute = await tool.execute({ model: path.join(project, "models", "marts", "customers.sql") }, ctx) + expect(absolute.metadata.success).toBe(true) + expect(absolute.metadata.file).toBe(path.join("models", "marts", "customers.sql")) + }) + + test("a unique_id the manifest does not hold is refused, never read as a name", async () => { + const tool = await initTool(DbtCteProfileTool) + const other = await tool.execute({ model: "model.other_package.customers" }, ctx) + expect(other.metadata.success).toBe(false) + expect(other.output).toContain("is not in target/manifest.json") + // The manifest knows this model but its file is gone: the name search must not answer for another package's. + const gone = await tool.execute({ model: "model.jaffle_shop.gone" }, ctx) + expect(gone.metadata.success).toBe(false) + // A package's model is recorded relative to its own package: never read as the project's file of that path. + const packaged = await tool.execute({ model: "model.pkg.customers" }, ctx) + expect(packaged.metadata.success).toBe(false) + expect(packaged.output).toContain("belongs to the package pkg") + }) + + test("a folder, or a path that is not a .sql file, is a failure, not a crash", async () => { + const tool = await initTool(DbtCteProfileTool) + const folder = await tool.execute({ model: "models/marts/" }, ctx) + expect(folder.metadata.success).toBe(false) + expect(folder.output).toContain("not a .sql file") + const config = await tool.execute({ model: "models/../dbt_project.yml" }, ctx) + expect(config.metadata.success).toBe(false) + expect(config.output).toContain("not a .sql file") + const dirNamedSql = path.join(project, "models", "dir.sql") + await mkdir(dirNamedSql, { recursive: true }) + const named = await tool.execute({ model: "models/dir.sql" }, ctx) + expect(named.metadata.success).toBe(false) + await rm(dirNamedSql, { recursive: true }) + }) + + test("a symlink out of the project is refused", async () => { + const outside = await mkdtemp(path.join(os.tmpdir(), "dbt-cte-outside-")) + await writeFile(path.join(outside, "secret.sql"), "with s as (select 1) select * from s") + await symlink(path.join(outside, "secret.sql"), path.join(project, "models", "link.sql")) + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "models/link.sql" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("outside the dbt project") + await rm(path.join(project, "models", "link.sql")) + await rm(outside, { recursive: true, force: true }) + }) + + test("the list is read from the session's project, not the one the adapter was set up for", async () => { + const other = await mkdtemp(path.join(os.tmpdir(), "dbt-cte-other-")) + await writeFile(path.join(other, "dbt_project.yml"), "name: other\n") + await mkdir(path.join(other, "models"), { recursive: true }) + await writeFile(path.join(other, "models", "customers.sql"), "with only_here as (select 1) select * from only_here") + const before = process.cwd() + process.chdir(other) + try { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers" }, ctx) + expect((result.metadata.ctes as { name: string }[]).map((c) => c.name)).toEqual(["only_here"]) + const run = await tool.execute({ model: "customers", cte: "only_here" }, ctx) + expect(run.metadata.success).toBe(false) + expect(run.metadata.refusal).toBe("other_project") + } finally { + process.chdir(before) + await rm(other, { recursive: true, force: true }) + } + }) + + test("an oversize model is refused", async () => { + await writeFile(path.join(project, "models", "huge.sql"), "-- " + "x".repeat(4 * 1024 * 1024 + 10)) + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "huge" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("largest file read") + await rm(path.join(project, "models", "huge.sql")) + }) +}) + +describe("dbt_cte_profile same-named CTEs", () => { + test("two CTEs whose names differ only in case are listed apart, and each is reachable by its exact name", async () => { + const tool = await initTool(DbtCteProfileTool) + const list = await tool.execute({ model: "twins" }, ctx) + expect((list.metadata.ctes as { name: string; position: number }[]).map((c) => [c.name, c.position])).toEqual([['"Foo"', 0], ["foo", 1], ["keep", 2]]) + const exact = await tool.execute({ model: "twins", cte: "foo" }, ctx) + expect(exact.metadata.position).toBe(1) + expect(queries[0].sql).toEndWith("FROM foo") + const quoted = await tool.execute({ model: "twins", cte: '"Foo"' }, ctx) + expect(quoted.metadata.position).toBe(0) + expect(queries[1].sql).toEndWith('FROM "Foo"') + // Letter case alone does not pick between two matches. + const loose = await tool.execute({ model: "twins", cte: "FOO" }, ctx) + expect(loose.metadata.success).toBe(false) + expect(loose.output).toContain("positions 0, 1") + expect(queries).toHaveLength(2) + }) + + test("position picks one CTE, and a name that disagrees with it is refused", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "twins", position: 2 }, ctx) + expect(result.metadata.cte).toBe("keep") + const cased = await tool.execute({ model: "twins", cte: "KEEP", position: 2 }, ctx) + expect(cased.metadata.success).toBe(true) + const wrong = await tool.execute({ model: "twins", cte: "foo", position: 2 }, ctx) + expect(wrong.metadata.success).toBe(false) + const missing = await tool.execute({ model: "twins", position: 9 }, ctx) + expect(missing.metadata.success).toBe(false) + expect(queries).toHaveLength(2) + }) + + test("a CTE of a second WITH clause has no query of its own and runs nothing", async () => { + const tool = await initTool(DbtCteProfileTool) + const first = await tool.execute({ model: "second", cte: "a" }, ctx) + expect(first.metadata.success).toBe(true) + const later = await tool.execute({ model: "second", cte: "b" }, ctx) + expect(later.metadata.success).toBe(false) + expect(later.output).toContain("second WITH clause") + expect(queries).toHaveLength(1) + }) +}) + +describe("dbt_cte_profile run", () => { + test("every failure carries an error, so the CLI and MCP report it as one", async () => { + const tool = await initTool(DbtCteProfileTool) + const results = [ + await tool.execute({ model: "nope" }, ctx), + await tool.execute({ model: "customers", cte: "nope" }, ctx), + await tool.execute({ model: "customers", cte: "orders" }, { ...ctx, extra: { forceReadOnly: true } }), + ] + for (const result of results) { + expect(result.metadata.success).toBe(false) + expect(typeof result.metadata.error).toBe("string") + } + }) + + test("a query that outlives the call's abort ends the wait at the abort", async () => { + respond = () => new Promise(() => {}) + const controller = new AbortController() + const tool = await initTool(DbtCteProfileTool) + const run = tool.execute({ model: "customers", cte: "orders" }, { ...ctx, abort: controller.signal }) + setTimeout(() => controller.abort(new Error("stopped")), 50) + await expect(run).rejects.toThrow("stopped") + }) + + test("a name shared by several versions of one model is refused, not guessed", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "sales" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("matches 2 models") + }) + + test("a query that timed out is reported as timed out, since it may still be running", async () => { + respond = async () => { + throw Object.assign(new Error("exceeded 300000 ms"), { code: "QUERY_TIMEOUT" }) + } + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.metadata.timedOut).toBe(true) + expect(result.output).toContain("may still be running") + }) + + test("a CTE inside a Jinja block that the query would cut open is refused, and nothing runs", async () => { + const tool = await initTool(DbtCteProfileTool) + for (const cte of ["a", "b"]) { + const result = await tool.execute({ model: "wrapped", cte }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("Jinja") + } + expect(queries).toHaveLength(0) + }) + + test("runs one count query over the CTE and the earlier ones, limit 1, and reads the count by position", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "Big Spenders" }, ctx) + expect(result.metadata.success).toBe(true) + expect(result.metadata.cte).toBe('"Big Spenders"') + expect(result.metadata.rows).toBe(42) + expect(typeof result.metadata.queryMs).toBe("number") + expect(queries).toHaveLength(1) + expect(queries[0].name).toBe('cte_profiler_"Big Spenders"') + expect(queries[0].limit).toBe(1) + expect(queries[0].sql).toContain("{{ config(materialized='table') }}") + expect(queries[0].sql).toContain("orders as (") + expect(queries[0].sql).toContain('"Big Spenders" (customer_id, total) as (') + expect(queries[0].sql).not.toContain("final as (") + expect(queries[0].sql).toEndWith('SELECT COUNT(*) AS _profile_count FROM "Big Spenders"') + }) + + test("matches the CTE name ignoring case and quotes", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "ORDERS" }, ctx) + expect(result.metadata.success).toBe(true) + expect(result.metadata.cte).toBe("orders") + expect(queries[0].sql).toEndWith("FROM orders") + }) + + test("an unknown CTE names the ones the model has and runs nothing", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "nope" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("orders") + expect(queries).toEqual([]) + }) + + test("a read-only call is refused and never starts the adapter", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "orders" }, { ...ctx, extra: { forceReadOnly: true } }) + expect(result.metadata.success).toBe(false) + expect(result.metadata.refusal).toBe("read_only") + expect(result.output).toContain("--allow-write") + expect(queries).toEqual([]) + }) + + test("a failing query is reported as a failure, not thrown", async () => { + respond = async () => { + throw new Error("Object 'STG_ORDERS' does not exist") + } + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("STG_ORDERS") + }) + + test("an empty result counts as zero rows", async () => { + respond = async () => ({ columnNames: ["_PROFILE_COUNT"], data: [] }) + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "final" }, ctx) + expect(result.metadata.rows).toBe(0) + }) + + test("a cancelled call throws instead of answering", async () => { + const controller = new AbortController() + controller.abort() + const tool = await initTool(DbtCteProfileTool) + await expect(tool.execute({ model: "customers", cte: "orders" }, { ...ctx, abort: controller.signal })).rejects.toMatchObject({ name: "AbortError" }) + expect(queries).toEqual([]) + }) +})