diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index 7a03aeb906..cb53c0ec7b 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -347,6 +347,26 @@ export namespace AltimateApi { // altimate_change end } + /** + * Calls an Altimate API endpoint with the stored credentials (Bearer key and x-tenant headers) and + * returns the parsed JSON body. Endpoint functions in other files go through this. Fails with a + * message naming the credentials file when none are stored or they can't be read. + */ + export async function apiRequest(method: string, endpoint: string, body?: unknown): Promise { + let creds: AltimateCredentials + try { + creds = await getCredentials() + } catch (e) { + const p = credentialsPath() + if (!(await Filesystem.exists(p))) + throw new Error( + `No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${p}).`, + ) + throw new Error(`The Altimate credentials in ${p} can't be used: ${e instanceof Error ? e.message : String(e)}`) + } + return request(creds, method, endpoint, body) + } + export async function listDatamates(creds?: AltimateCredentials) { creds ??= await getCredentials() const data = await request(creds, "GET", "/datamates/") diff --git a/packages/opencode/src/altimate/api/sql-lineage.ts b/packages/opencode/src/altimate/api/sql-lineage.ts new file mode 100644 index 0000000000..89f6c55073 --- /dev/null +++ b/packages/opencode/src/altimate/api/sql-lineage.ts @@ -0,0 +1,49 @@ +import { AltimateApi } from "./client" + +/** One parent relation handed to the backend: its warehouse location and its columns. */ +export type SqlLineageModelNode = { + database: string + schema: string + name: string + alias: string + uniqueId: string + columns: Record + path: string | undefined +} + +export type SqlLineageRequest = { + compiled_sql: string + model_info: { model_node: SqlLineageModelNode }[] + model_dialect: string + session_id: string +} + +/** + * One node of the backend's graph, keyed in `details` by an id that edges use. A CTE or the final select carries `sql`; a relation, join, filter, group, sort or + * subquery carries `expression`. A relation, subquery, CTE and the final select have a `name`; a join, filter, group or sort has none. + */ +export type SqlLineageNodeDetails = { + name?: string + type: string + nodeType?: string + nodeId?: string + sql?: string + expression?: string + /** A relation's warehouse database (catalog) and schema. */ + db?: string + schema?: string + columns?: { name: string; datatype?: string; expression?: string }[] +} + +export type SqlLineageResponse = { + tableEdges: [string, string][] + details: Record + nodePositions?: Record +} + +export const SQL_LINEAGE_ENDPOINT = "/dbt/v3/sql_lineage" + +/** The backend's graph of a compiled model's CTEs, joins and filters. */ +export async function sqlLineage(req: SqlLineageRequest): Promise { + return (await AltimateApi.apiRequest("POST", SQL_LINEAGE_ENDPOINT, req)) as SqlLineageResponse +} diff --git a/packages/opencode/src/altimate/tool-cli/allow-list.ts b/packages/opencode/src/altimate/tool-cli/allow-list.ts index 33b452d75c..0bb4f18473 100644 --- a/packages/opencode/src/altimate/tool-cli/allow-list.ts +++ b/packages/opencode/src/altimate/tool-cli/allow-list.ts @@ -35,10 +35,19 @@ import { AltimateCoreTestgenTool } from "../tools/altimate-core-testgen" import { AltimateCoreTrackLineageTool } from "../tools/altimate-core-track-lineage" import { AltimateCoreValidateTool } from "../tools/altimate-core-validate" import { DataDiffTool } from "../tools/data-diff" +import { DbtCostEstimateTool } from "../tools/dbt-cost-estimate" +import { DbtCteProfileTool } from "../tools/dbt-cte-profile" +import { DbtDeferStateTool } from "../tools/dbt-defer-state" +import { DbtDocsCoverageTool } from "../tools/dbt-docs-coverage" +import { DbtHealthTool } from "../tools/dbt-health" import { DbtLineageTool } from "../tools/dbt-lineage" import { DbtManifestTool } from "../tools/dbt-manifest" +import { DbtMapRelationsTool } from "../tools/dbt-map-relations" import { DbtProfilesTool } from "../tools/dbt-profiles" +import { DbtRunResultsTool } from "../tools/dbt-run-results" +import { DbtSqlGraphTool } from "../tools/dbt-sql-graph" import { DbtUnitTestGenTool } from "../tools/dbt-unit-test-gen" +import { DocsShareCommentsTool } from "../tools/docs-share-comments" import { FinopsAnalyzeCreditsTool } from "../tools/finops-analyze-credits" import { FinopsExpensiveQueriesTool } from "../tools/finops-expensive-queries" import { FinopsQueryHistoryTool } from "../tools/finops-query-history" @@ -106,10 +115,19 @@ export const EXPOSED: Record = { altimate_core_track_lineage: AltimateCoreTrackLineageTool, altimate_core_validate: AltimateCoreValidateTool, data_diff: DataDiffTool, + dbt_cost_estimate: DbtCostEstimateTool, + dbt_cte_profile: DbtCteProfileTool, + dbt_defer_state: DbtDeferStateTool, + dbt_docs_coverage: DbtDocsCoverageTool, + dbt_health: DbtHealthTool, dbt_lineage: DbtLineageTool, dbt_manifest: DbtManifestTool, + dbt_map_relations: DbtMapRelationsTool, dbt_profiles: DbtProfilesTool, + dbt_run_results: DbtRunResultsTool, + dbt_sql_graph: DbtSqlGraphTool, dbt_unit_test_gen: DbtUnitTestGenTool, + docs_share_comments: DocsShareCommentsTool, finops_analyze_credits: FinopsAnalyzeCreditsTool, finops_expensive_queries: FinopsExpensiveQueriesTool, finops_query_history: FinopsQueryHistoryTool, diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts new file mode 100644 index 0000000000..2f5d48b924 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -0,0 +1,103 @@ +import fs from "fs" +import path from "path" +import type { Tool } from "../../tool/tool" +import { projectRoot } from "../native/connections/registry" + +/** Why a tool may not use the dbt adapter. */ +export type DbtAdapterRefusal = "read_only" | "other_project" | "no_project" | "not_started" | "not_loaded" + +/** The project's dbt adapter, or why this call may not use it (`reason` is a sentence for the user). */ +export type DbtAdapterAccess = + | { adapter: any; refusal?: undefined; reason?: undefined } + | { adapter?: undefined; refusal: DbtAdapterRefusal; reason: string } + +/** Test seams. */ +export const dbtAdapterAccessInternals = { + /** How long a call waits for the dbt adapter to start before it gives up; the start goes on for the next call. */ + adapterStartTimeoutMs: 120_000, +} + +/** + * The dbt adapter for a tool call, which every adapter-backed tool goes through. Starting the adapter + * runs dbt deps and writes target/, and compiling renders Jinja, which can run SQL: a read-only call + * (no write permission, or plan mode) never starts or uses it. Nor does a session in a dbt project the + * adapter isn't set up for. The wait for the adapter to start is bounded and follows the call's abort. + */ +export async function dbtAdapterFor(ctx: Pick): Promise { + const signal = ctx.abort + // A cancelled call is cancelled, not refused. + signal.throwIfAborted() + if (ctx.extra?.forceReadOnly === true || ctx.agent === "plan") + return { + refusal: "read_only", + reason: "this needs the dbt adapter, which a read-only call doesn't start: re-run with --allow-write", + } + try { + const { getDbtAdapter, getDbtProjectRoot } = await import("../native/connections/register") + // The import yields: a call cancelled meanwhile starts nothing. + signal.throwIfAborted() + const project = await sessionProject() + signal.throwIfAborted() + if (project?.other) { + return { + refusal: "other_project", + reason: project.root + ? `the dbt adapter is set up for ${project.root}, not ${project.other}` + : `the dbt adapter isn't set up for ${project.other}`, + } + } + let stop = () => {} + let timer: ReturnType | undefined + const waited = new Promise((_, reject) => { + stop = () => reject(signal.reason) + signal.addEventListener("abort", stop, { once: true }) + const seconds = dbtAdapterAccessInternals.adapterStartTimeoutMs / 1000 + timer = setTimeout( + () => reject(new Error(`the dbt adapter didn't start within ${seconds} s`)), + dbtAdapterAccessInternals.adapterStartTimeoutMs, + ) + }) + const adapter = await Promise.race([getDbtAdapter(), waited]).finally(() => { + signal.removeEventListener("abort", stop) + clearTimeout(timer) + }) + if (!adapter) { + // No adapter also when the project exists but the adapter failed to start: say which. + const root = await getDbtProjectRoot() + const found = !!root && fs.existsSync(path.join(root, "dbt_project.yml")) + return found + ? { refusal: "not_started", reason: "the dbt adapter didn't start" } + : { refusal: "no_project", reason: "no dbt project here" } + } + if (adapter.isInitialized?.() === false) + return { refusal: "not_loaded", reason: "the dbt adapter didn't load the project" } + return { adapter } + } catch (e) { + if (signal.aborted) throw e + return { refusal: "not_started", reason: e instanceof Error ? e.message : String(e) } + } +} + +/** + * The adapter's project root and, when the session's directory is in another dbt project, that one + * (`other`, both links resolved). Undefined when the session's directory is in no dbt project: the + * adapter's own discovery (a dbt.json, or a walk up from the working directory) decides then. + */ +async function sessionProject(): Promise<{ root?: string; other?: string } | undefined> { + const { findProjectRoot } = await import("../../../../dbt-tools/src/config") + const found = findProjectRoot(projectRoot()) + if (!found) return undefined + const { getDbtProjectRoot } = await import("../native/connections/register") + const root = await getDbtProjectRoot() + const real = (p: string) => { + try { + return fs.realpathSync(p) + } catch { + return path.resolve(p) + } + } + const here = real(found) + // path.relative ignores case on win32. + if (!root) return { other: here } + return path.relative(real(root), here) ? { root: real(root), other: here } : { root } +} diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts new file mode 100644 index 0000000000..1b113f1994 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -0,0 +1,17 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtCostEstimateTool = Tool.define("dbt_cost_estimate", { + description: "Estimate the bytes a dbt model or a SQL query would scan, from a warehouse dry run (BigQuery).", + parameters: z.object({ + model: z.string().optional().describe("Model name to compile and estimate"), + sql: z.string().optional().describe("SQL (may use Jinja) to compile and estimate"), + }), + async execute() { + return { + title: "dbt_cost_estimate: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_cost_estimate is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts new file mode 100644 index 0000000000..0e4913f06d --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -0,0 +1,17 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { + description: "List a dbt model's CTEs, or time one CTE with a row-count query.", + parameters: z.object({ + model: z.string().describe("Model name"), + cte: z.string().optional().describe("CTE to time; without it, the CTE list is returned"), + }), + async execute() { + return { + title: "dbt_cte_profile: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_cte_profile is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-defer-state.ts b/packages/opencode/src/altimate/tools/dbt-defer-state.ts new file mode 100644 index 0000000000..9df344ec1b --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-defer-state.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtDeferStateTool = Tool.define("dbt_defer_state", { + description: "dbt Power User's defer-to-production setting for a dbt project, and the profile's default target.", + parameters: z.object({ + project_dir: z.string().describe("dbt project folder"), + }), + async execute() { + return { + title: "dbt_defer_state: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_defer_state is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts b/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts new file mode 100644 index 0000000000..8990306b72 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts @@ -0,0 +1,21 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtDocsCoverageTool = Tool.define("dbt_docs_coverage", { + description: + "Documentation coverage of a dbt model: which output columns have a description, and whether the model has one.", + parameters: z.object({ + model: z.string().describe("Model name"), + manifest_path: z + .string() + .optional() + .describe("manifest.json to read (default: the dbt project's target/manifest.json)"), + }), + async execute() { + return { + title: "dbt_docs_coverage: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_docs_coverage is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-health.ts b/packages/opencode/src/altimate/tools/dbt-health.ts new file mode 100644 index 0000000000..096e2bc3f4 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-health.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtHealthTool = Tool.define("dbt_health", { + description: "DataPilot project health insights for the dbt project.", + parameters: z.object({ + config_path: z.string().optional().describe("DataPilot config file to use"), + }), + async execute() { + return { + title: "dbt_health: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_health is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-map-relations.ts b/packages/opencode/src/altimate/tools/dbt-map-relations.ts new file mode 100644 index 0000000000..f949274620 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-map-relations.ts @@ -0,0 +1,21 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtMapRelationsTool = Tool.define("dbt_map_relations", { + description: "Map each table a raw SQL query reads to the dbt ref() or source() for it, from the manifest.", + parameters: z.object({ + sql: z.string().describe("Raw SQL to map"), + dialect: z.string().optional().describe("SQL dialect of the query"), + manifest_path: z + .string() + .optional() + .describe("manifest.json to match against (default: the dbt project's target/manifest.json)"), + }), + async execute() { + return { + title: "dbt_map_relations: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_map_relations is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-run-results.ts b/packages/opencode/src/altimate/tools/dbt-run-results.ts new file mode 100644 index 0000000000..c6f2cf396c --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-run-results.ts @@ -0,0 +1,19 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtRunResultsTool = Tool.define("dbt_run_results", { + description: "The latest dbt run's results from target/run_results.json: command, duration and each node's status.", + parameters: z.object({ + project_dir: z + .string() + .optional() + .describe("dbt project folder (default: the dbt project found from the working directory)"), + }), + async execute() { + return { + title: "dbt_run_results: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "dbt_run_results is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts new file mode 100644 index 0000000000..40a39987d2 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts @@ -0,0 +1,343 @@ +import z from "zod" +import crypto from "crypto" +import fs from "fs" +import path from "path" +import { Tool } from "../../tool/tool" +import { + sqlLineage, + type SqlLineageModelNode, + type SqlLineageRequest, + type SqlLineageResponse, +} from "../api/sql-lineage" +import { dbtAdapterFor } from "./dbt-adapter-access" + +const FINAL_SELECT = "__final_select__" + +export type SqlGraphNode = { + /** Unique in the graph; the edges name nodes by it. */ + id: string + /** What the node is called on screen; two nodes can share one (a relation used twice). */ + name: string + /** The backend's node type ("cte", "table", ...); a relation that maps to a dbt node carries its resource type ("model", "source", ...). */ + type: string + nodeId?: string + sql: string + columns: { name: string; datatype?: string; expression?: string }[] +} + +export type SqlGraph = { nodes: SqlGraphNode[]; edges: [string, string][] } + +type ManifestNode = { + unique_id: string + name: string + alias: string + resource_type: string + path?: string + database: string + schema: string + columns: SqlLineageModelNode["columns"] +} +type ManifestSource = { + name: string + database: string + schema: string + tables: { name: string; identifier: string; path?: string; columns: SqlLineageModelNode["columns"] }[] +} + +/** The slice of dbt-integration's adapter this tool uses. */ +export type SqlGraphAdapter = { + parseManifest(): Promise< + | { + nodeMetaMap: { + lookupByUniqueId(id: string): ManifestNode | undefined + lookupByBaseName(name: string): ManifestNode | undefined + } + sourceMetaMap: { get(name: string): ManifestSource | undefined } + } + | undefined + > + unsafeCompileQuery(query: string, originalModelName?: string): Promise + getNonEphemeralParents(keys: string[]): string[] + getBulkSchemaFromDB( + nodes: { unique_id: string; name: string; resource_type: string; table?: string }[], + signal: AbortSignal, + ): Promise> + getAdapterType(): string +} + +export type SqlGraphDeps = { + fetchLineage: (req: SqlLineageRequest) => Promise + readFile: (file: string) => string + /** The dbt project's root, to resolve a model's project-relative path. */ + projectRoot: string | undefined + /** Names the caller's session in the backend's session id, with the SQL's hash. */ + sessionId: string +} + +/** Adapters whose unquoted identifiers fold to lower case; the rest fold to upper case. */ +const LOWER_FOLDING = new Set(["trino", "athena", "postgres", "redshift", "duckdb", "risingwave"]) + +/** A name the warehouse would only keep as written if it were quoted: it is not in the adapter's own folding case. */ +function isQuotedIdentifier(name: string, adapter: string): boolean { + return LOWER_FOLDING.has(adapter) ? !/^([_a-z]+[_a-z0-9$]*)$/.test(name) : !/^([_A-Z]+[_A-Z0-9$]*)$/.test(name) +} + +/** A column name as dbt Power User keys it: lower case unless the adapter would only keep it as written when quoted. */ +function columnKey(name: string, adapter: string): string { + return isQuotedIdentifier(name, adapter) ? name : name.toLowerCase() +} + +/** The parent's manifest columns plus any the warehouse reports that the manifest lacks. */ +export function mergeColumns( + manifestColumns: SqlLineageModelNode["columns"], + fromDb: { column: string; dtype: string }[] | undefined, + adapter: string, +): SqlLineageModelNode["columns"] { + // A Map: a column named `constructor` or `__proto__` is a column, not an inherited property. + const merged = new Map() + for (const [k, v] of Object.entries(manifestColumns ?? {})) merged.set(columnKey(k, adapter), { ...v }) + for (const c of fromDb ?? []) { + const key = columnKey(c.column, adapter) + const existing = merged.get(key) + if (existing) { + existing.data_type = (existing.data_type || c.dtype)?.toLowerCase() + continue + } + merged.set(key, { name: key, data_type: c.dtype?.toLowerCase(), description: "", meta: {} }) + } + return Object.fromEntries(merged) +} + +/** + * The backend names the model's own select `__final_select__`: name it after the model (or ` (final select)` when the SQL has a node of that name), tie each + * relation node to the dbt node it stands for, and drop the layout positions the list doesn't use. + */ +export function remapSqlLineage( + response: SqlLineageResponse, + model: { name: string; uniqueId: string; resourceType: string }, + parents: { + uniqueId: string + alias?: string + sourceTable?: string + schema?: string + database?: string + resourceType: string + }[], +): SqlGraph { + const details = { ...response.details } + const mapping: Record = {} + // A relation that more than one parent could be is tied to none: an arbitrary dbt node would be shown as the real one. + const claimed = new Map>() + for (const p of parents) { + // A source's relation is its table's identifier; any other node's, its alias. + const wanted = (p.sourceTable ?? p.alias ?? "").toLowerCase() + if (!wanted) continue + // The backend keys a relation by its location and a per-occurrence suffix, so the match is on its name, in the parent's database and schema when both say one; + // every occurrence of the relation in the SQL stands for the same dbt node. + for (const [k, d] of Object.entries(details)) { + if (d.type !== "table" || (d.name ?? k).toLowerCase() !== wanted) continue + if (d.schema && p.schema && d.schema.toLowerCase() !== p.schema.toLowerCase()) continue + if (d.db && p.database && d.db.toLowerCase() !== p.database.toLowerCase()) continue + claimed.set(k, (claimed.get(k) ?? new Set()).add(p.uniqueId)) + mapping[k] = { nodeId: p.uniqueId, type: p.resourceType } + } + } + for (const [k, owners] of claimed) if (owners.size > 1) delete mapping[k] + // A CTE the SQL names like the model (`with customers as (…)`) keeps its node: the model's own select is told apart. + const finalKey = Object.hasOwn(details, model.name) ? `${model.name} (final select)` : model.name + mapping[finalKey] = { nodeId: model.uniqueId, type: model.resourceType } + const edges = response.tableEdges.map( + (edge) => edge.map((item) => (item === FINAL_SELECT ? finalKey : item)) as [string, string], + ) + if (details[FINAL_SELECT]) { + details[finalKey] = details[FINAL_SELECT] + delete details[FINAL_SELECT] + } + const nodes: SqlGraphNode[] = Object.entries(details).map(([key, d]) => { + const merged = { ...d, ...mapping[key] } + return { + // The key is the node's identity (edges use it); the name is only what is shown. + id: key, + name: key === finalKey ? finalKey : (merged.name ?? key), + type: merged.type, + ...(merged.nodeId ? { nodeId: merged.nodeId } : {}), + sql: merged.sql ?? merged.expression ?? "", + columns: merged.columns ?? [], + } + }) + return { nodes, edges } +} + +/** The promise's result, or an abort as soon as the signal fires: the backend call itself is bounded by the API client's own timeout. */ +function untilAborted(work: Promise, signal: AbortSignal): Promise { + if (signal.aborted) { + work.catch(() => {}) + return Promise.reject(signal.reason) + } + return new Promise((resolve, reject) => { + const onAbort = () => reject(signal.reason) + signal.addEventListener("abort", onAbort, { once: true }) + work.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort)) + }) +} + +/** Compile the model, read its parents' columns, ask the backend, and return the remapped graph. */ +export async function buildSqlGraph( + adapter: SqlGraphAdapter, + model: string, + deps: SqlGraphDeps, + signal: AbortSignal, +): Promise { + const manifest = await adapter.parseManifest() + signal.throwIfAborted() + if (!manifest) throw new Error("the dbt manifest is not available") + const node = manifest.nodeMetaMap.lookupByUniqueId(model) ?? manifest.nodeMetaMap.lookupByBaseName(model) + if (!node) throw new Error(`no model named ${model} in the manifest`) + // A seed, snapshot or analysis shares the node map: only a model's .sql file is compiled and sent. + if (node.resource_type !== "model" || !node.path?.toLowerCase().endsWith(".sql")) + throw new Error(`${node.name} is not a dbt model with a SQL file`) + const file = path.isAbsolute(node.path) ? node.path : path.join(deps.projectRoot ?? "", node.path) + const compiled = await adapter.unsafeCompileQuery(deps.readFile(file), node.name) + signal.throwIfAborted() + if (!compiled) throw new Error(`unable to compile ${node.name}`) + + const parentIds = adapter.getNonEphemeralParents([node.unique_id]) + const requests: { unique_id: string; name: string; resource_type: string; table?: string }[] = [] + const infos: SqlLineageModelNode[] = [] + const parents: { + uniqueId: string + alias?: string + sourceTable?: string + schema?: string + database?: string + resourceType: string + }[] = [] + for (const id of parentIds) { + const parts = id.split(".") + if (parts[0] === "source") { + const source = manifest.sourceMetaMap.get(parts[2]) + const table = source?.tables.find((t) => t.name === parts[3]) + if (!source || !table) continue + requests.push({ unique_id: id, name: source.name, resource_type: "source", table: table.name }) + infos.push({ + database: source.database, + schema: source.schema, + name: table.name, + alias: table.identifier, + uniqueId: id, + columns: table.columns, + path: table.path, + }) + parents.push({ + uniqueId: id, + sourceTable: table.identifier || table.name, + schema: source.schema, + database: source.database, + resourceType: "source", + }) + continue + } + const parent = manifest.nodeMetaMap.lookupByUniqueId(id) + if (!parent) continue + requests.push({ unique_id: id, name: parent.name, resource_type: parent.resource_type }) + infos.push({ + database: parent.database, + schema: parent.schema, + name: parent.name, + alias: parent.alias, + uniqueId: id, + columns: parent.columns, + path: parent.path, + }) + parents.push({ + uniqueId: id, + alias: parent.alias, + schema: parent.schema, + database: parent.database, + resourceType: parent.resource_type, + }) + } + // The warehouse metadata read: once per call, only for the parents. + const fromDb = requests.length ? await adapter.getBulkSchemaFromDB(requests, signal) : {} + signal.throwIfAborted() + const hash = crypto.createHash("md5").update(compiled).digest("hex") + const dialect = adapter.getAdapterType() + const response = await untilAborted( + deps.fetchLineage({ + compiled_sql: compiled, + model_info: infos.map((n) => ({ + model_node: { ...n, columns: mergeColumns(n.columns, fromDb[n.uniqueId], dialect) }, + })), + model_dialect: dialect, + session_id: `${deps.sessionId}-${hash}`, + }), + signal, + ) + signal.throwIfAborted() + if (!Array.isArray(response?.tableEdges) || !response.details) throw new Error("the backend returned no SQL graph") + return remapSqlLineage( + response, + { name: node.name, uniqueId: node.unique_id, resourceType: node.resource_type }, + parents, + ) +} + +/** The graph as a list, one line per node: its type, its column count and its inputs. */ +export function formatSqlGraph(model: string, graph: SqlGraph): string { + const inputs = new Map() + const nameOf = new Map(graph.nodes.map((n) => [n.id, n.name])) + for (const [from, to] of graph.edges) inputs.set(to, [...(inputs.get(to) ?? []), nameOf.get(from) ?? from]) + const lines = [`SQL graph for ${model}: ${graph.nodes.length} nodes, ${graph.edges.length} edges`] + for (const n of graph.nodes) { + const from = inputs.get(n.id) + lines.push( + `- ${n.name} (${n.type}${n.nodeId ? `, ${n.nodeId}` : ""}): ${n.columns.length} columns${from?.length ? `, from ${from.join(", ")}` : ""}`, + ) + } + return lines.join("\n") +} + +export const DbtSqlGraphTool = Tool.define("dbt_sql_graph", { + description: + "The CTEs, joins and filters of a dbt model as a graph, from the Altimate backend: nodes with their SQL and columns, and the edges between them. Compiles the model with the project's dbt integration, reads its parent relations' columns from the warehouse's metadata, and sends the compiled SQL to the Altimate backend. Needs write permission (starts the dbt integration) and an Altimate key.", + parameters: z.object({ + model: z.string().describe("Model name or unique_id (e.g. 'orders', 'model.shop.orders')"), + }), + async execute(args, ctx) { + const access = await dbtAdapterFor(ctx) + if (!access.adapter) { + return { + title: "dbt SQL graph: not available", + metadata: { success: false, model: args.model, reason: access.reason, error: access.reason }, + output: `Not available: ${access.reason}`, + } + } + try { + const { getDbtProjectRoot } = await import("../native/connections/register") + const graph = await buildSqlGraph( + access.adapter as SqlGraphAdapter, + args.model, + { + fetchLineage: sqlLineage, + readFile: (f) => fs.readFileSync(f, "utf8"), + projectRoot: await getDbtProjectRoot(), + sessionId: ctx.sessionID, + }, + ctx.abort, + ) + return { + title: `dbt SQL graph: ${args.model} (${graph.nodes.length} nodes)`, + metadata: { success: true, model: args.model, nodes: graph.nodes, edges: graph.edges }, + output: formatSqlGraph(args.model, graph), + } + } catch (e) { + if (ctx.abort.aborted) throw e + const msg = e instanceof Error ? e.message : String(e) + return { + title: "dbt SQL graph: ERROR", + metadata: { success: false, model: args.model, error: msg }, + output: `Failed: ${msg}`, + } + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/docs-share-comments.ts b/packages/opencode/src/altimate/tools/docs-share-comments.ts new file mode 100644 index 0000000000..8963268cdd --- /dev/null +++ b/packages/opencode/src/altimate/tools/docs-share-comments.ts @@ -0,0 +1,20 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DocsShareCommentsTool = Tool.define("docs_share_comments", { + description: "Comment threads on shared dbt docs: list open threads, reply to one, or resolve one.", + parameters: z.object({ + action: z.enum(["list", "reply", "resolve"]).describe("What to do"), + project: z.string().optional().describe("dbt project name the shared docs belong to"), + share_id: z.number().int().optional().describe("Shared docs id"), + group_id: z.number().int().optional().describe("Comment thread id"), + text: z.string().optional().describe("Reply text"), + }), + async execute() { + return { + title: "docs_share_comments: not implemented", + metadata: { success: false, error: "not implemented" }, + output: "docs_share_comments is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts new file mode 100644 index 0000000000..ca7b0adb96 --- /dev/null +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -0,0 +1,280 @@ +// The tools the dbt Power User features call: their allow-list entries and placeholder answers, the +// one gate every dbt-adapter-backed tool goes through, and the API helper for new endpoint files. The +// dbt adapter is replaced through `dbtAdapterInternals`, so these tests need no Python or dbt. +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { realpathSync } from "node:fs" +import os from "node:os" +import path from "node:path" +import { Effect } from "effect" +import type { Tool } from "@/tool/tool" +import { SessionID, MessageID } from "@/session/schema" +import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" +import { dbtAdapterAccessInternals, dbtAdapterFor } from "@/altimate/tools/dbt-adapter-access" +import { EXPOSED, MCP_OPEN_WRITE_TOOLS, WRITE_TOOLS } from "@/altimate/tool-cli/allow-list" +import { AltimateApi } from "@/altimate/api/client" +import { Instance } from "@/project/instance" +import { disposeInstance } from "@/effect/instance-registry" +import { initTool } from "./tool-fixture" + +/** Each tool with the smallest arguments its parameters accept. */ +const MINIMAL_ARGS: Record> = { + dbt_run_results: {}, + dbt_defer_state: { project_dir: "." }, + dbt_map_relations: { sql: "select 1" }, + docs_share_comments: { action: "list" }, + dbt_cost_estimate: {}, + dbt_health: {}, + dbt_sql_graph: { model: "customers" }, + dbt_docs_coverage: { model: "customers" }, + dbt_cte_profile: { model: "customers" }, +} +const FEATURE_TOOLS = Object.keys(MINIMAL_ARGS) +/** Tools whose package has replaced the placeholder; they have tests of their own. */ +const IMPLEMENTED = new Set(["dbt_sql_graph"]) + +const ctx: Tool.Context = { + sessionID: SessionID.make("ses_dbt_adapter_access"), + messageID: MessageID.make("msg_dbt_adapter_access"), + agent: "analyst", + abort: new AbortController().signal, + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, +} + +let project: string + +function useConfig() { + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: project, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) +} + +beforeAll(async () => { + project = await mkdtemp(path.join(os.tmpdir(), "dbt-adapter-access-")) + await writeFile(path.join(project, "dbt_project.yml"), "name: jaffle_shop\n") +}) + +afterAll(async () => { + await rm(project, { recursive: true, force: true }) +}) + +beforeEach(() => resetDbtAdapter()) + +afterEach(() => { + resetDbtAdapter() + delete dbtAdapterInternals.readConfig + delete dbtAdapterInternals.create +}) + +describe("feature tools in the allow-list", () => { + test("all nine are exposed, in the map's alphabetical order, and none is a write tool", () => { + const ids = Object.keys(EXPOSED) + for (const id of FEATURE_TOOLS) { + expect(ids).toContain(id) + expect(WRITE_TOOLS.has(id)).toBe(false) + expect(MCP_OPEN_WRITE_TOOLS.has(id)).toBe(false) + } + const listed = ids.filter((id) => id.startsWith("dbt_") || id.startsWith("docs_")) + expect(listed).toEqual([...listed].sort()) + }) + + test("each placeholder answers that it isn't implemented, as a failure", async () => { + for (const id of FEATURE_TOOLS.filter((one) => !IMPLEMENTED.has(one))) { + const tool = await initTool(EXPOSED[id] as any) + expect(tool.id).toBe(id) + const result = await tool.execute(MINIMAL_ARGS[id], ctx) + expect(result.title).toBe(`${id}: not implemented`) + expect(result.metadata).toEqual({ success: false, error: "not implemented", truncated: false }) + expect(result.output).toContain(id) + } + }) +}) + +describe("dbtAdapterFor", () => { + let created: number + beforeEach(() => { + created = 0 + useConfig() + dbtAdapterInternals.create = async () => { + created += 1 + return { isInitialized: () => true, dispose: async () => {} } + } + }) + + test("refuses a read-only call and never creates the adapter", async () => { + const access = await dbtAdapterFor({ ...ctx, extra: { forceReadOnly: true } }) + expect(access.adapter).toBeUndefined() + expect(access.refusal).toBe("read_only") + expect(access.reason).toContain("--allow-write") + expect(created).toBe(0) + }) + + test("a read-only call cancelled beforehand is cancelled, not refused", async () => { + const controller = new AbortController() + controller.abort() + await expect( + dbtAdapterFor({ ...ctx, abort: controller.signal, extra: { forceReadOnly: true } }), + ).rejects.toThrow() + expect(created).toBe(0) + }) + + test("refuses a plan-mode call", async () => { + const access = await dbtAdapterFor({ ...ctx, agent: "plan" }) + expect(access.refusal).toBe("read_only") + expect(created).toBe(0) + }) + + test("returns the adapter where writes are allowed", async () => { + const access = await dbtAdapterFor(ctx) + expect(access.refusal).toBeUndefined() + expect(access.adapter?.isInitialized()).toBe(true) + expect(created).toBe(1) + }) + + test("says when the adapter didn't load the project", async () => { + dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_loaded", reason: "the dbt adapter didn't load the project" }) + }) + + test("says there is no dbt project when the config finds none", async () => { + dbtAdapterInternals.readConfig = async () => null + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "no_project", reason: "no dbt project here" }) + }) + + test("says when the adapter doesn't start in time", async () => { + dbtAdapterInternals.create = () => new Promise(() => {}) + const saved = dbtAdapterAccessInternals.adapterStartTimeoutMs + dbtAdapterAccessInternals.adapterStartTimeoutMs = 20 + try { + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start within 0.02 s" }) + } finally { + dbtAdapterAccessInternals.adapterStartTimeoutMs = saved + } + }) + + test("says the adapter didn't start when the project exists but creating it fails", async () => { + dbtAdapterInternals.create = async () => { + created += 1 + throw new Error("dbt-integration missing") + } + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start" }) + expect(created).toBe(1) + }) + + test("a call cancelled while the adapter starts throws and stops waiting", async () => { + let entered = () => {} + const starting = new Promise((resolve) => (entered = resolve)) + dbtAdapterInternals.create = () => { + entered() + return new Promise(() => {}) + } + const controller = new AbortController() + const pending = dbtAdapterFor({ ...ctx, abort: controller.signal }) + await starting + controller.abort(new Error("cancelled")) + await expect(pending).rejects.toThrow("cancelled") + }) + + test("a cancelled call throws rather than answering", async () => { + const controller = new AbortController() + controller.abort() + await expect(dbtAdapterFor({ ...ctx, abort: controller.signal })).rejects.toThrow() + expect(created).toBe(0) + }) + + test("a session in a dbt project the adapter isn't set up for says so", async () => { + dbtAdapterInternals.readConfig = async () => null + try { + const access = await Instance.provide({ directory: project, fn: () => dbtAdapterFor(ctx) }) + expect(access).toEqual({ + refusal: "other_project", + reason: `the dbt adapter isn't set up for ${realpathSync(project)}`, + }) + expect(created).toBe(0) + } finally { + await disposeInstance(project) + } + }) + + test("a session in another dbt project than the adapter's says which", async () => { + const other = await mkdtemp(path.join(os.tmpdir(), "dbt-adapter-other-")) + await writeFile(path.join(other, "dbt_project.yml"), "name: other\n") + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: other, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) + try { + const access = await Instance.provide({ directory: project, fn: () => dbtAdapterFor(ctx) }) + expect(access).toEqual({ + refusal: "other_project", + reason: `the dbt adapter is set up for ${realpathSync(other)}, not ${realpathSync(project)}`, + }) + expect(created).toBe(0) + } finally { + await disposeInstance(project) + await rm(other, { recursive: true, force: true }) + } + }) +}) + +describe("AltimateApi.apiRequest", () => { + let home: string + let saved: string | undefined + beforeEach(async () => { + home = await mkdtemp(path.join(os.tmpdir(), "altimate-api-home-")) + saved = process.env.OPENCODE_TEST_HOME + process.env.OPENCODE_TEST_HOME = home + }) + afterEach(async () => { + if (saved === undefined) delete process.env.OPENCODE_TEST_HOME + else process.env.OPENCODE_TEST_HOME = saved + await rm(home, { recursive: true, force: true }) + }) + + test("fails with a clear message when no credentials are stored", async () => { + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow( + `No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${path.join(home, ".altimate", "altimate.json")}).`, + ) + }) + + test("sends the stored key and tenant and returns the JSON body", async () => { + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: "https://api.example.test/", altimateInstanceName: "acme", altimateApiKey: "k1" }), + ) + const original = globalThis.fetch + const seen: { url: string; init: RequestInit }[] = [] + globalThis.fetch = (async (url: string, init: RequestInit) => { + seen.push({ url, init }) + return new Response(JSON.stringify({ ok: 1 }), { status: 200 }) + }) as unknown as typeof fetch + try { + expect(await AltimateApi.apiRequest("POST", "/dbt/v1/x", { a: 1 })).toEqual({ ok: 1 }) + expect(seen[0].url).toBe("https://api.example.test/dbt/v1/x") + const headers = seen[0].init.headers as Record + expect(headers.Authorization).toBe("Bearer k1") + expect(headers["x-tenant"]).toBe("acme") + expect(seen[0].init.body).toBe(JSON.stringify({ a: 1 })) + } finally { + globalThis.fetch = original + } + }) + + test("names the file when the stored credentials can't be read", async () => { + await Bun.write(path.join(home, ".altimate", "altimate.json"), "{not json") + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow( + `The Altimate credentials in ${path.join(home, ".altimate", "altimate.json")} can't be used`, + ) + }) +}) diff --git a/packages/opencode/test/altimate/dbt-sql-graph.test.ts b/packages/opencode/test/altimate/dbt-sql-graph.test.ts new file mode 100644 index 0000000000..8717cae54b --- /dev/null +++ b/packages/opencode/test/altimate/dbt-sql-graph.test.ts @@ -0,0 +1,715 @@ +// altimate_change - new file +// +// dbt_sql_graph: the request built from the compiled model and its parents' columns, and the +// remap of the backend's reply (`__final_select__` to the model's node, relation nodes to dbt unique ids). +// The adapter and the endpoint are fakes, so no dbt, warehouse or backend is involved. +import { afterEach, describe, expect, test } from "bun:test" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { realpathSync } from "node:fs" +import os from "node:os" +import path from "node:path" +import { Effect } from "effect" +import { SessionID, MessageID } from "@/session/schema" +import type { Tool } from "@/tool/tool" +import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" +import { initTool } from "./tool-fixture" +import { + buildSqlGraph, + formatSqlGraph, + mergeColumns, + remapSqlLineage, + DbtSqlGraphTool, + type SqlGraphAdapter, +} from "@/altimate/tools/dbt-sql-graph" +import type { SqlLineageRequest, SqlLineageResponse } from "@/altimate/api/sql-lineage" + +const col = (name: string, data_type = "") => ({ name, description: "", data_type, meta: {} }) + +const orders = { + unique_id: "model.shop.orders", + name: "orders", + alias: "orders", + resource_type: "model", + path: "/proj/models/orders.sql", + database: "db", + schema: "analytics", + columns: {}, +} +const stgCustomers = { + unique_id: "model.shop.stg_customers", + name: "stg_customers", + alias: "stg_customers", + resource_type: "model", + path: "/proj/models/stg_customers.sql", + database: "db", + schema: "analytics", + columns: { id: col("id", "int") }, +} +const rawOrders = { + name: "raw", + database: "db", + schema: "raw", + tables: [{ name: "raw_orders", identifier: "RAW_ORDERS_T", columns: {} }], +} + +function adapter(overrides: Partial = {}): SqlGraphAdapter & { calls: string[] } { + const calls: string[] = [] + const nodes = [orders, stgCustomers] + return { + calls, + parseManifest: async () => ({ + nodeMetaMap: { + lookupByUniqueId: (id) => nodes.find((n) => n.unique_id === id), + lookupByBaseName: (name) => nodes.find((n) => n.name === name), + }, + sourceMetaMap: { get: (name) => (name === "raw" ? rawOrders : undefined) }, + }), + unsafeCompileQuery: async (q) => { + calls.push(`compile:${q}`) + return "with o as (select * from raw.orders) select * from o join stg_customers c on 1=1" + }, + getNonEphemeralParents: () => ["source.shop.raw.raw_orders", "model.shop.stg_customers"], + getBulkSchemaFromDB: async (requested) => { + calls.push(`schema:${requested.map((r) => r.unique_id).join(",")}`) + return { + "source.shop.raw.raw_orders": [ + { column: "ID", dtype: "INT" }, + { column: "AMOUNT", dtype: "NUMBER" }, + ], + "model.shop.stg_customers": [ + { column: "ID", dtype: "INT" }, + { column: "NAME", dtype: "TEXT" }, + ], + } + }, + getAdapterType: () => "snowflake", + ...overrides, + } +} + +const reply: SqlLineageResponse = { + tableEdges: [ + ["raw_orders", "o"], + ["o", "__final_select__"], + ["stg_customers", "__final_select__"], + ], + details: { + raw_orders: { name: "RAW_ORDERS_T", type: "table", columns: [{ name: "id" }, { name: "amount" }] }, + o: { name: "o", type: "cte", sql: "select * from raw.orders", columns: [{ name: "id" }] }, + stg_customers: { name: "stg_customers", type: "table", sql: "", columns: [{ name: "id" }, { name: "name" }] }, + __final_select__: { name: "__final_select__", type: "final", sql: "select * from o", columns: [{ name: "id" }] }, + }, + nodePositions: { o: [1, 1], __final_select__: [2, 2] }, +} + +const signal = new AbortController().signal + +describe("mergeColumns", () => { + test("adds warehouse columns the manifest lacks, lower-cased, and fills a missing type", () => { + const merged = mergeColumns( + { id: col("id") }, + [ + { column: "ID", dtype: "INT" }, + { column: "NEW_ONE", dtype: "TEXT" }, + ], + "snowflake", + ) + expect(Object.keys(merged)).toEqual(["id", "new_one"]) + expect(merged.id.data_type).toBe("int") + expect(merged.new_one).toEqual({ name: "new_one", data_type: "text", description: "", meta: {} }) + }) + + test("keeps a quoted name as written and tolerates no warehouse answer", () => { + expect(Object.keys(mergeColumns({}, [{ column: '"MixedCase"', dtype: "int" }], "snowflake"))).toEqual([ + '"MixedCase"', + ]) + expect(mergeColumns({ a: col("a", "int") }, undefined, "snowflake").a.data_type).toBe("int") + }) + + test("a name outside the adapter's folding case stays distinct: Postgres Foo and foo, Snowflake Foo and FOO", () => { + const pg = mergeColumns( + {}, + [ + { column: "Foo", dtype: "int" }, + { column: "foo", dtype: "text" }, + ], + "postgres", + ) + expect(Object.keys(pg)).toEqual(["Foo", "foo"]) + expect(pg.Foo.data_type).toBe("int") + expect(pg.foo.data_type).toBe("text") + const sf = mergeColumns( + {}, + [ + { column: "Foo", dtype: "int" }, + { column: "FOO", dtype: "text" }, + ], + "snowflake", + ) + expect(Object.keys(sf)).toEqual(["Foo", "foo"]) + }) + + test("Redshift folds to lower case like Postgres: FOO and foo stay distinct columns", () => { + const rs = mergeColumns( + {}, + [ + { column: "FOO", dtype: "int" }, + { column: "foo", dtype: "text" }, + ], + "redshift", + ) + expect(Object.keys(rs)).toEqual(["FOO", "foo"]) + expect(rs.FOO.data_type).toBe("int") + expect(rs.foo.data_type).toBe("text") + }) + + test("a manifest column and the warehouse's name for it meet under one key in the adapter's case", () => { + const merged = mergeColumns({ id: col("id") }, [{ column: "id", dtype: "BIGINT" }], "postgres") + expect(Object.keys(merged)).toEqual(["id"]) + expect(merged.id.data_type).toBe("bigint") + }) + + test("columns named like Object members are columns, and nothing is written to the built-ins", () => { + const merged = mergeColumns( + {}, + [ + { column: "constructor", dtype: "text" }, + { column: "__proto__", dtype: "int" }, + ], + "postgres", + ) + expect(Object.keys(merged)).toEqual(["constructor", "__proto__"]) + expect(Object.getOwnPropertyDescriptor(merged, "constructor")?.value.data_type).toBe("text") + expect((Object.prototype as any).data_type).toBeUndefined() + expect((Object as any).data_type).toBeUndefined() + }) +}) + +describe("remapSqlLineage", () => { + const parents = [ + { uniqueId: "source.shop.raw.raw_orders", sourceTable: "RAW_ORDERS_T", resourceType: "source" }, + { uniqueId: "model.shop.stg_customers", alias: "STG_CUSTOMERS", resourceType: "model" }, + ] + const graph = remapSqlLineage( + reply, + { name: "orders", uniqueId: "model.shop.orders", resourceType: "model" }, + parents, + ) + + test("renames __final_select__ to the model in edges and nodes", () => { + expect(graph.edges).toEqual([ + ["raw_orders", "o"], + ["o", "orders"], + ["stg_customers", "orders"], + ]) + expect(graph.nodes.map((n) => n.id)).toEqual(["raw_orders", "o", "stg_customers", "orders"]) + expect(JSON.stringify(graph)).not.toContain("__final_select__") + }) + + test("ties table nodes to dbt unique ids, case-insensitively, and the model to itself", () => { + const byId = Object.fromEntries(graph.nodes.map((n) => [n.id, n])) + expect(byId.raw_orders).toEqual({ + id: "raw_orders", + name: "RAW_ORDERS_T", + type: "source", + nodeId: "source.shop.raw.raw_orders", + sql: "", + columns: [{ name: "id" }, { name: "amount" }], + }) + expect(byId.stg_customers).toEqual({ + id: "stg_customers", + name: "stg_customers", + type: "model", + nodeId: "model.shop.stg_customers", + sql: "", + columns: [{ name: "id" }, { name: "name" }], + }) + expect(byId.orders).toEqual({ + id: "orders", + name: "orders", + type: "model", + nodeId: "model.shop.orders", + sql: "select * from o", + columns: [{ name: "id" }], + }) + }) + + test("leaves CTEs unmapped and drops the layout positions", () => { + const cte = graph.nodes.find((n) => n.id === "o")! + expect(cte).toEqual({ id: "o", name: "o", type: "cte", sql: "select * from raw.orders", columns: [{ name: "id" }] }) + expect(graph).not.toHaveProperty("nodePositions") + }) + + test("a CTE or table named like the model keeps its node; the model's own select is told apart", () => { + const collide: SqlLineageResponse = { + tableEdges: [ + ["stg_customers", "orders"], + ["orders", "__final_select__"], + ], + details: { + stg_customers: { name: "stg_customers", type: "table", sql: "", columns: [] }, + orders: { name: "orders", type: "cte", sql: "select * from stg_customers", columns: [{ name: "id" }] }, + __final_select__: { name: "__final_select__", type: "final", sql: "select * from orders", columns: [] }, + }, + } + const out = remapSqlLineage(collide, { name: "orders", uniqueId: "model.shop.orders", resourceType: "model" }, []) + expect(out.edges).toEqual([ + ["stg_customers", "orders"], + ["orders", "orders (final select)"], + ]) + expect(out.nodes).toEqual([ + { id: "stg_customers", name: "stg_customers", type: "table", sql: "", columns: [] }, + { id: "orders", name: "orders", type: "cte", sql: "select * from stg_customers", columns: [{ name: "id" }] }, + { + id: "orders (final select)", + name: "orders (final select)", + type: "model", + nodeId: "model.shop.orders", + sql: "select * from orders", + columns: [], + }, + ]) + }) + + test("a source table named like the model stays the source's node; a model named like an Object member is not renamed", () => { + const sameName: SqlLineageResponse = { + tableEdges: [["orders", "__final_select__"]], + details: { + orders: { name: "orders", type: "table", sql: "", columns: [] }, + __final_select__: { name: "__final_select__", type: "final", sql: "select * from orders", columns: [] }, + }, + } + const parentsOf = [{ uniqueId: "source.shop.raw.orders", sourceTable: "orders", resourceType: "source" }] + const out = remapSqlLineage( + sameName, + { name: "orders", uniqueId: "model.shop.orders", resourceType: "model" }, + parentsOf, + ) + expect(out.nodes.map((n) => [n.name, n.nodeId])).toEqual([ + ["orders", "source.shop.raw.orders"], + ["orders (final select)", "model.shop.orders"], + ]) + const member = remapSqlLineage( + { tableEdges: [["x", "__final_select__"]], details: { __final_select__: sameName.details.__final_select__ } }, + { name: "constructor", uniqueId: "model.shop.constructor", resourceType: "model" }, + [], + ) + expect(member.nodes.map((n) => n.name)).toEqual(["constructor"]) + expect(member.edges).toEqual([["x", "constructor"]]) + }) + + test("does not mutate the backend reply", () => { + expect(reply.details.__final_select__).toBeDefined() + expect(reply.tableEdges[1]).toEqual(["o", "__final_select__"]) + }) +}) + +describe("buildSqlGraph", () => { + const read = (file: string) => `-- ${file}\nselect 1` + + test("sends the compiled SQL, parents' merged columns and dialect, then remaps the reply", async () => { + const a = adapter() + let sent: SqlLineageRequest | undefined + const graph = await buildSqlGraph( + a, + "orders", + { + fetchLineage: async (req) => { + sent = req + return reply + }, + readFile: read, + projectRoot: "/proj", + sessionId: "ses_1", + }, + signal, + ) + expect(a.calls).toEqual([ + "compile:-- /proj/models/orders.sql\nselect 1", + "schema:source.shop.raw.raw_orders,model.shop.stg_customers", + ]) + expect(sent?.model_dialect).toBe("snowflake") + expect(sent?.compiled_sql).toContain("join stg_customers") + expect(sent?.session_id).toMatch(/^ses_1-[0-9a-f]{32}$/) + expect(sent?.model_info).toEqual([ + { + model_node: { + database: "db", + schema: "raw", + name: "raw_orders", + alias: "RAW_ORDERS_T", + uniqueId: "source.shop.raw.raw_orders", + columns: { + id: { name: "id", data_type: "int", description: "", meta: {} }, + amount: { name: "amount", data_type: "number", description: "", meta: {} }, + }, + path: undefined, + }, + }, + { + model_node: { + database: "db", + schema: "analytics", + name: "stg_customers", + alias: "stg_customers", + uniqueId: "model.shop.stg_customers", + columns: { + id: { name: "id", description: "", data_type: "int", meta: {} }, + name: { name: "name", data_type: "text", description: "", meta: {} }, + }, + path: "/proj/models/stg_customers.sql", + }, + }, + ]) + expect(graph.edges).toContainEqual(["o", "orders"]) + expect(graph.nodes.find((n) => n.name === "orders")?.nodeId).toBe("model.shop.orders") + expect(graph.nodes.find((n) => n.id === "raw_orders")?.nodeId).toBe("source.shop.raw.raw_orders") + }) + + test("accepts a unique_id and resolves a project-relative path against the project root", async () => { + const relative = { ...orders, path: "models/orders.sql" } + const a = adapter({ + parseManifest: async () => ({ + nodeMetaMap: { + lookupByUniqueId: (id) => (id === relative.unique_id ? relative : undefined), + lookupByBaseName: () => undefined, + }, + sourceMetaMap: { get: () => undefined }, + }), + getNonEphemeralParents: () => [], + }) + let read: string | undefined + await buildSqlGraph( + a, + "model.shop.orders", + { + fetchLineage: async () => reply, + readFile: (f) => ((read = f), "select 1"), + projectRoot: "/proj", + sessionId: "ses_1", + }, + signal, + ) + expect(read).toBe("/proj/models/orders.sql") + // no parents: no warehouse read + expect(a.calls.some((c) => c.startsWith("schema:"))).toBe(false) + }) + + test("an unknown model, an uncompilable model and a missing manifest each fail with a reason", async () => { + const deps = { fetchLineage: async () => reply, readFile: read, projectRoot: "/proj", sessionId: "ses_1" } + await expect(buildSqlGraph(adapter(), "nope", deps, signal)).rejects.toThrow("no model named nope") + await expect( + buildSqlGraph(adapter({ unsafeCompileQuery: async () => undefined }), "orders", deps, signal), + ).rejects.toThrow("unable to compile orders") + await expect( + buildSqlGraph(adapter({ parseManifest: async () => undefined }), "orders", deps, signal), + ).rejects.toThrow("manifest is not available") + }) + + test("a seed, a snapshot or a model with no .sql file is refused before anything is compiled", async () => { + const deps = { fetchLineage: async () => reply, readFile: read, projectRoot: "/proj", sessionId: "ses_1" } + for (const odd of [ + { ...orders, name: "countries", resource_type: "seed", path: "seeds/countries.csv" }, + { ...orders, name: "snap", resource_type: "snapshot", path: "snapshots/snap.sql" }, + { ...orders, name: "py_model", path: "models/py_model.py" }, + ]) { + const a = adapter({ + parseManifest: async () => ({ + nodeMetaMap: { lookupByUniqueId: () => undefined, lookupByBaseName: () => odd }, + sourceMetaMap: { get: () => undefined }, + }), + }) + await expect(buildSqlGraph(a, odd.name, deps, signal)).rejects.toThrow(`${odd.name} is not a dbt model`) + expect(a.calls).toEqual([]) + } + }) + + test("a cancel while the manifest parses or the model compiles starts no further work", async () => { + const deps = { fetchLineage: async () => reply, readFile: read, projectRoot: "/proj", sessionId: "ses_1" } + const duringParse = new AbortController() + const a = adapter() + const parse = a.parseManifest + a.parseManifest = async () => { + const manifest = await parse() + duringParse.abort() + return manifest + } + await expect(buildSqlGraph(a, "orders", deps, duringParse.signal)).rejects.toThrow() + expect(a.calls).toEqual([]) + + const duringCompile = new AbortController() + const b = adapter({ + unsafeCompileQuery: async () => { + duringCompile.abort() + return "select 1" + }, + }) + await expect(buildSqlGraph(b, "orders", deps, duringCompile.signal)).rejects.toThrow() + expect(b.calls).toEqual([]) + }) + + test("two sessions asking for the same SQL send different session ids", async () => { + const ids: string[] = [] + const fetchLineage = async (req: SqlLineageRequest) => (ids.push(req.session_id), reply) + const base = { fetchLineage, readFile: read, projectRoot: "/proj" } + await buildSqlGraph(adapter(), "orders", { ...base, sessionId: "ses_a" }, signal) + await buildSqlGraph(adapter(), "orders", { ...base, sessionId: "ses_b" }, signal) + expect(ids[0]).not.toBe(ids[1]) + expect(ids[0]?.split("-").slice(-1)[0]).toBe(ids[1]?.split("-").slice(-1)[0]) + }) + + test("a backend failure propagates and sends nothing after a cancel", async () => { + const deps = { + fetchLineage: async () => { + throw new Error("API POST /dbt/v3/sql_lineage failed with status 500") + }, + readFile: read, + projectRoot: "/proj", + sessionId: "ses_1", + } + await expect(buildSqlGraph(adapter(), "orders", deps, signal)).rejects.toThrow("status 500") + const aborted = AbortSignal.abort() + let called = false + await expect( + buildSqlGraph(adapter(), "orders", { ...deps, fetchLineage: async () => ((called = true), reply) }, aborted), + ).rejects.toThrow() + expect(called).toBe(false) + }) +}) + +describe("formatSqlGraph", () => { + test("lists each node with its inputs and column count", () => { + const graph = remapSqlLineage(reply, { name: "orders", uniqueId: "model.shop.orders", resourceType: "model" }, []) + const text = formatSqlGraph("orders", graph) + expect(text).toContain("4 nodes, 3 edges") + expect(text).toContain("- orders (model, model.shop.orders): 1 columns, from o, stg_customers") + expect(text).toContain("- o (cte): 1 columns, from RAW_ORDERS_T") + }) +}) + +describe("the backend's real node shapes", () => { + // Relations are keyed by location and a per-occurrence suffix, and carry `name`, `db` and `schema`; joins, filters and the like have no name and carry + // `expression` instead of `sql`. + const real: SqlLineageResponse = { + tableEdges: [ + ["analytics.customers_111", "LEFT_JOIN_9"], + ["raw.customers_222", "LEFT_JOIN_9"], + ["LEFT_JOIN_9", "__final_select__"], + ["analytics.customers_111", "WHERE_7"], + ["WHERE_7", "__final_select__"], + ], + details: { + "analytics.customers_111": { + name: "customers", + type: "table", + expression: "analytics.customers", + db: "db", + schema: "analytics", + columns: [], + }, + "raw.customers_222": { + name: "customers", + type: "table", + expression: "raw.customers", + db: "db", + schema: "raw", + columns: [], + }, + LEFT_JOIN_9: { type: "LEFT_JOIN", expression: "LEFT JOIN raw.customers ON 1 = 1", columns: [] }, + WHERE_7: { type: "FILTER", expression: "WHERE amount > 0" }, + __final_select__: { name: "__final_select__", type: "final", sql: "select 1", columns: [{ name: "id" }] }, + }, + } + const graph = remapSqlLineage(real, { name: "orders", uniqueId: "model.shop.orders", resourceType: "model" }, [ + { uniqueId: "model.shop.customers", alias: "customers", schema: "analytics", resourceType: "model" }, + { uniqueId: "source.shop.raw.customers", sourceTable: "customers", schema: "raw", resourceType: "source" }, + ]) + + test("every edge ends at a node id, though two relations share a name", () => { + const ids = graph.nodes.map((n) => n.id) + expect(new Set(ids).size).toBe(ids.length) + for (const edge of graph.edges) for (const end of edge) expect(ids).toContain(end) + expect(graph.nodes.filter((n) => n.name === "customers")).toHaveLength(2) + }) + + test("a relation is tied to the dbt node in its own schema, and a join or filter keeps its SQL", () => { + const byId = Object.fromEntries(graph.nodes.map((n) => [n.id, n])) + expect(byId["analytics.customers_111"]).toMatchObject({ + nodeId: "model.shop.customers", + type: "model", + sql: "analytics.customers", + }) + expect(byId["raw.customers_222"]).toMatchObject({ nodeId: "source.shop.raw.customers", type: "source" }) + expect(byId.LEFT_JOIN_9).toEqual({ + id: "LEFT_JOIN_9", + name: "LEFT_JOIN_9", + type: "LEFT_JOIN", + sql: "LEFT JOIN raw.customers ON 1 = 1", + columns: [], + }) + expect(byId.WHERE_7.sql).toBe("WHERE amount > 0") + }) + + test("two relations of one name and schema in different databases are told apart by the database", () => { + const twoDbs: SqlLineageResponse = { + tableEdges: [], + details: { + "a.raw.customers_1": { name: "customers", type: "table", db: "a", schema: "raw" }, + "b.raw.customers_2": { name: "customers", type: "table", db: "b", schema: "raw" }, + }, + } + const parentsIn = (first: string, second: string) => [ + { + uniqueId: `source.p.${first}.customers`, + sourceTable: "customers", + schema: "raw", + database: first, + resourceType: "source", + }, + { + uniqueId: `source.p.${second}.customers`, + sourceTable: "customers", + schema: "raw", + database: second, + resourceType: "source", + }, + ] + const model = { name: "orders", uniqueId: "model.shop.orders", resourceType: "model" } + for (const order of [parentsIn("a", "b"), parentsIn("b", "a")]) { + const out = remapSqlLineage(twoDbs, model, order) + expect(out.nodes.map((n) => n.nodeId)).toEqual(["source.p.a.customers", "source.p.b.customers"]) + } + }) + + test("a relation two parents could both be is tied to neither, whatever the parent order", () => { + const ambiguous: SqlLineageResponse = { + tableEdges: [], + details: { "raw.customers_1": { name: "customers", type: "table", schema: "raw" } }, + } + const model = { name: "orders", uniqueId: "model.shop.orders", resourceType: "model" } + const a = { uniqueId: "source.p.raw.customers", sourceTable: "customers", schema: "raw", resourceType: "source" } + const b = { uniqueId: "model.p.customers", alias: "customers", schema: "raw", resourceType: "model" } + for (const order of [ + [a, b], + [b, a], + ]) { + const out = remapSqlLineage(ambiguous, model, order) + expect(out.nodes).toEqual([{ id: "raw.customers_1", name: "customers", type: "table", sql: "", columns: [] }]) + } + }) + + test("every occurrence of one relation in the SQL is tied to its dbt node", () => { + const twice: SqlLineageResponse = { + tableEdges: [], + details: { + "analytics.customers_1": { name: "customers", type: "table", schema: "analytics" }, + "analytics.customers_2": { name: "customers", type: "table", schema: "analytics" }, + }, + } + const out = remapSqlLineage(twice, { name: "orders", uniqueId: "model.shop.orders", resourceType: "model" }, [ + { uniqueId: "model.shop.customers", alias: "customers", schema: "analytics", resourceType: "model" }, + ]) + expect(out.nodes.map((n) => n.nodeId)).toEqual(["model.shop.customers", "model.shop.customers"]) + }) +}) + +describe("a cancel while the backend is answering", () => { + test("ends the call at once, and a late answer is ignored", async () => { + const controller = new AbortController() + let release!: (r: SqlLineageResponse) => void + let reached!: () => void + const inBackend = new Promise((resolve) => (reached = resolve)) + const slow = new Promise((resolve) => (release = resolve)) + const call = buildSqlGraph( + adapter(), + "orders", + { + fetchLineage: () => { + reached() + return slow + }, + readFile: (f) => f, + projectRoot: "/proj", + sessionId: "ses_1", + }, + controller.signal, + ) + const settled = call.then( + () => "answered", + () => "aborted", + ) + await inBackend + controller.abort() + expect(await settled).toBe("aborted") + release(reply) + }) + + test("a reply with no graph is a clear failure, not a crash", async () => { + const deps = { + fetchLineage: async () => ({}) as SqlLineageResponse, + readFile: (f: string) => f, + projectRoot: "/proj", + sessionId: "ses_1", + } + await expect(buildSqlGraph(adapter(), "orders", deps, signal)).rejects.toThrow("the backend returned no SQL graph") + }) +}) + +describe("the tool's own answer", () => { + const ctx: Tool.Context = { + sessionID: SessionID.make("ses_sql_graph"), + messageID: MessageID.make("msg_sql_graph"), + agent: "analyst", + abort: new AbortController().signal, + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, + } + + afterEach(() => { + resetDbtAdapter() + delete dbtAdapterInternals.readConfig + delete dbtAdapterInternals.create + }) + + test("a read-only call is a failure the CLI reports as one: success false and an error", async () => { + const tool = await initTool(DbtSqlGraphTool as any) + const result = await tool.execute({ model: "orders" }, { ...ctx, extra: { forceReadOnly: true } }) + expect(result.metadata).toEqual({ + success: false, + model: "orders", + reason: expect.stringContaining("--allow-write"), + error: expect.stringContaining("--allow-write"), + truncated: false, + }) + expect(result.output).toContain("Not available") + }) + + test("a model the manifest lacks is a failure with the reason", async () => { + const project = realpathSync(await mkdtemp(path.join(os.tmpdir(), "dbt-sql-graph-"))) + await writeFile(path.join(project, "dbt_project.yml"), "name: shop\n") + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: project, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) + dbtAdapterInternals.create = async () => ({ + isInitialized: () => true, + dispose: async () => {}, + parseManifest: async () => ({ + nodeMetaMap: { lookupByUniqueId: () => undefined, lookupByBaseName: () => undefined }, + sourceMetaMap: { get: () => undefined }, + }), + }) + const tool = await initTool(DbtSqlGraphTool as any) + const result = await tool + .execute({ model: "nope" }, ctx) + .finally(() => rm(project, { recursive: true, force: true })) + expect(result.metadata).toMatchObject({ + success: false, + model: "nope", + error: "no model named nope in the manifest", + }) + }) +})