From b7cfc0426d60963a83f22a8f5c6053c7b7fce8e4 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 16:30:43 +0530 Subject: [PATCH 1/8] feat: foundation for the dbt Power User tools Nine tool stubs exposed through `altimate tool run` (dbt_run_results, dbt_defer_state, dbt_map_relations, docs_share_comments, dbt_cost_estimate, dbt_health, dbt_sql_graph, dbt_docs_coverage, dbt_cte_profile), an exported `AltimateApi.apiRequest` that calls the backend with the stored credentials, and `dbtAdapterFor`, the one gate every adapter-backed tool goes through: it refuses read-only calls, other projects, and an adapter that did not start. Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/api/client.ts | 18 ++ .../src/altimate/tool-cli/allow-list.ts | 18 ++ .../src/altimate/tools/dbt-adapter-access.ts | 100 ++++++++ .../src/altimate/tools/dbt-cost-estimate.ts | 17 ++ .../src/altimate/tools/dbt-cte-profile.ts | 17 ++ .../src/altimate/tools/dbt-defer-state.ts | 16 ++ .../src/altimate/tools/dbt-docs-coverage.ts | 17 ++ .../opencode/src/altimate/tools/dbt-health.ts | 16 ++ .../src/altimate/tools/dbt-map-relations.ts | 18 ++ .../src/altimate/tools/dbt-run-results.ts | 16 ++ .../src/altimate/tools/dbt-sql-graph.ts | 16 ++ .../src/altimate/tools/docs-share-comments.ts | 20 ++ .../test/altimate/dbt-adapter-access.test.ts | 221 ++++++++++++++++++ 13 files changed, 510 insertions(+) create mode 100644 packages/opencode/src/altimate/tools/dbt-adapter-access.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-cost-estimate.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-cte-profile.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-defer-state.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-docs-coverage.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-health.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-map-relations.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-run-results.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-sql-graph.ts create mode 100644 packages/opencode/src/altimate/tools/docs-share-comments.ts create mode 100644 packages/opencode/test/altimate/dbt-adapter-access.test.ts diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index 7a03aeb906..069a182b7a 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -347,6 +347,24 @@ export namespace AltimateApi { // altimate_change end } + /** + * Calls an Altimate API endpoint with the stored credentials (Bearer key and x-tenant headers) and + * returns the parsed JSON body. Endpoint functions in other files go through this. Fails with a + * message naming the credentials file when none are stored or they can't be read. + */ + export async function apiRequest(method: string, endpoint: string, body?: unknown): Promise { + let creds: AltimateCredentials + try { + creds = await getCredentials() + } catch (e) { + const p = credentialsPath() + if (!(await Filesystem.exists(p))) + throw new Error(`No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${p}).`) + throw new Error(`The Altimate credentials in ${p} can't be used: ${e instanceof Error ? e.message : String(e)}`) + } + return request(creds, method, endpoint, body) + } + export async function listDatamates(creds?: AltimateCredentials) { creds ??= await getCredentials() const data = await request(creds, "GET", "/datamates/") diff --git a/packages/opencode/src/altimate/tool-cli/allow-list.ts b/packages/opencode/src/altimate/tool-cli/allow-list.ts index 33b452d75c..0bb4f18473 100644 --- a/packages/opencode/src/altimate/tool-cli/allow-list.ts +++ b/packages/opencode/src/altimate/tool-cli/allow-list.ts @@ -35,10 +35,19 @@ import { AltimateCoreTestgenTool } from "../tools/altimate-core-testgen" import { AltimateCoreTrackLineageTool } from "../tools/altimate-core-track-lineage" import { AltimateCoreValidateTool } from "../tools/altimate-core-validate" import { DataDiffTool } from "../tools/data-diff" +import { DbtCostEstimateTool } from "../tools/dbt-cost-estimate" +import { DbtCteProfileTool } from "../tools/dbt-cte-profile" +import { DbtDeferStateTool } from "../tools/dbt-defer-state" +import { DbtDocsCoverageTool } from "../tools/dbt-docs-coverage" +import { DbtHealthTool } from "../tools/dbt-health" import { DbtLineageTool } from "../tools/dbt-lineage" import { DbtManifestTool } from "../tools/dbt-manifest" +import { DbtMapRelationsTool } from "../tools/dbt-map-relations" import { DbtProfilesTool } from "../tools/dbt-profiles" +import { DbtRunResultsTool } from "../tools/dbt-run-results" +import { DbtSqlGraphTool } from "../tools/dbt-sql-graph" import { DbtUnitTestGenTool } from "../tools/dbt-unit-test-gen" +import { DocsShareCommentsTool } from "../tools/docs-share-comments" import { FinopsAnalyzeCreditsTool } from "../tools/finops-analyze-credits" import { FinopsExpensiveQueriesTool } from "../tools/finops-expensive-queries" import { FinopsQueryHistoryTool } from "../tools/finops-query-history" @@ -106,10 +115,19 @@ export const EXPOSED: Record = { altimate_core_track_lineage: AltimateCoreTrackLineageTool, altimate_core_validate: AltimateCoreValidateTool, data_diff: DataDiffTool, + dbt_cost_estimate: DbtCostEstimateTool, + dbt_cte_profile: DbtCteProfileTool, + dbt_defer_state: DbtDeferStateTool, + dbt_docs_coverage: DbtDocsCoverageTool, + dbt_health: DbtHealthTool, dbt_lineage: DbtLineageTool, dbt_manifest: DbtManifestTool, + dbt_map_relations: DbtMapRelationsTool, dbt_profiles: DbtProfilesTool, + dbt_run_results: DbtRunResultsTool, + dbt_sql_graph: DbtSqlGraphTool, dbt_unit_test_gen: DbtUnitTestGenTool, + docs_share_comments: DocsShareCommentsTool, finops_analyze_credits: FinopsAnalyzeCreditsTool, finops_expensive_queries: FinopsExpensiveQueriesTool, finops_query_history: FinopsQueryHistoryTool, diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts new file mode 100644 index 0000000000..a2711e35d1 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -0,0 +1,100 @@ +import fs from "fs" +import path from "path" +import type { Tool } from "../../tool/tool" +import { projectRoot } from "../native/connections/registry" + +/** Why a tool may not use the dbt adapter. */ +export type DbtAdapterRefusal = "read_only" | "other_project" | "no_project" | "not_started" | "not_loaded" + +/** The project's dbt adapter, or why this call may not use it (`reason` is a sentence for the user). */ +export type DbtAdapterAccess = + | { adapter: any; refusal?: undefined; reason?: undefined } + | { adapter?: undefined; refusal: DbtAdapterRefusal; reason: string } + +/** Test seams. */ +export const dbtAdapterAccessInternals = { + /** How long a call waits for the dbt adapter to start before it gives up; the start goes on for the next call. */ + adapterStartTimeoutMs: 120_000, +} + +/** + * The dbt adapter for a tool call, which every adapter-backed tool goes through. Starting the adapter + * runs dbt deps and writes target/, and compiling renders Jinja, which can run SQL: a read-only call + * (no write permission, or plan mode) never starts or uses it. Nor does a session in a dbt project the + * adapter isn't set up for. The wait for the adapter to start is bounded and follows the call's abort. + */ +export async function dbtAdapterFor(ctx: Pick): Promise { + const signal = ctx.abort + if (ctx.extra?.forceReadOnly === true || ctx.agent === "plan") + return { + refusal: "read_only", + reason: "this needs the dbt adapter, which a read-only call doesn't start: re-run with --allow-write", + } + try { + const { getDbtAdapter, getDbtProjectRoot } = await import("../native/connections/register") + // The import yields: a call cancelled meanwhile starts nothing. + signal.throwIfAborted() + const project = await sessionProject() + signal.throwIfAborted() + if (project?.other) { + return { + refusal: "other_project", + reason: project.root + ? `the dbt adapter is set up for ${project.root}, not ${project.other}` + : `the dbt adapter isn't set up for ${project.other}`, + } + } + let stop = () => {} + let timer: ReturnType | undefined + const waited = new Promise((_, reject) => { + stop = () => reject(signal.reason) + signal.addEventListener("abort", stop, { once: true }) + const seconds = dbtAdapterAccessInternals.adapterStartTimeoutMs / 1000 + timer = setTimeout( + () => reject(new Error(`the dbt adapter didn't start within ${seconds} s`)), + dbtAdapterAccessInternals.adapterStartTimeoutMs, + ) + }) + const adapter = await Promise.race([getDbtAdapter(), waited]).finally(() => { + signal.removeEventListener("abort", stop) + clearTimeout(timer) + }) + if (!adapter) { + // No adapter also when the project exists but the adapter failed to start: say which. + const root = await getDbtProjectRoot() + const found = !!root && fs.existsSync(path.join(root, "dbt_project.yml")) + return found + ? { refusal: "not_started", reason: "the dbt adapter didn't start" } + : { refusal: "no_project", reason: "no dbt project here" } + } + if (adapter.isInitialized?.() === false) return { refusal: "not_loaded", reason: "the dbt adapter didn't load the project" } + return { adapter } + } catch (e) { + if (signal.aborted) throw e + return { refusal: "not_started", reason: e instanceof Error ? e.message : String(e) } + } +} + +/** + * The adapter's project root and, when the session's directory is in another dbt project, that one + * (`other`, both links resolved). Undefined when the session's directory is in no dbt project: the + * adapter's own discovery (a dbt.json, or a walk up from the working directory) decides then. + */ +async function sessionProject(): Promise<{ root?: string; other?: string } | undefined> { + const { findProjectRoot } = await import("../../../../dbt-tools/src/config") + const found = findProjectRoot(projectRoot()) + if (!found) return undefined + const { getDbtProjectRoot } = await import("../native/connections/register") + const root = await getDbtProjectRoot() + const real = (p: string) => { + try { + return fs.realpathSync(p) + } catch { + return path.resolve(p) + } + } + const here = real(found) + // path.relative ignores case on win32. + if (!root) return { other: here } + return path.relative(real(root), here) ? { root: real(root), other: here } : { root } +} diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts new file mode 100644 index 0000000000..6529d828c6 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -0,0 +1,17 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtCostEstimateTool = Tool.define("dbt_cost_estimate", { + description: "Estimate the bytes a dbt model or a SQL query would scan, from a warehouse dry run (BigQuery).", + parameters: z.object({ + model: z.string().optional().describe("Model name to compile and estimate"), + sql: z.string().optional().describe("SQL (may use Jinja) to compile and estimate"), + }), + async execute() { + return { + title: "dbt_cost_estimate: not implemented", + metadata: { success: false }, + output: "dbt_cost_estimate is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts new file mode 100644 index 0000000000..e7a5bde647 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -0,0 +1,17 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { + description: "List a dbt model's CTEs, or time one CTE with a row-count query.", + parameters: z.object({ + model: z.string().describe("Model name"), + cte: z.string().optional().describe("CTE to time; without it, the CTE list is returned"), + }), + async execute() { + return { + title: "dbt_cte_profile: not implemented", + metadata: { success: false }, + output: "dbt_cte_profile is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-defer-state.ts b/packages/opencode/src/altimate/tools/dbt-defer-state.ts new file mode 100644 index 0000000000..75cbfe19cf --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-defer-state.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtDeferStateTool = Tool.define("dbt_defer_state", { + description: "dbt Power User's defer-to-production setting for a dbt project, and the profile's default target.", + parameters: z.object({ + project_dir: z.string().describe("dbt project folder"), + }), + async execute() { + return { + title: "dbt_defer_state: not implemented", + metadata: { success: false }, + output: "dbt_defer_state is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts b/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts new file mode 100644 index 0000000000..c4e0ec42d3 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts @@ -0,0 +1,17 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtDocsCoverageTool = Tool.define("dbt_docs_coverage", { + description: "Documentation coverage of a dbt model: which output columns have a description, and whether the model has one.", + parameters: z.object({ + model: z.string().describe("Model name"), + manifest_path: z.string().optional().describe("manifest.json to read (default: the dbt project's target/manifest.json)"), + }), + async execute() { + return { + title: "dbt_docs_coverage: not implemented", + metadata: { success: false }, + output: "dbt_docs_coverage is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-health.ts b/packages/opencode/src/altimate/tools/dbt-health.ts new file mode 100644 index 0000000000..a3f26352c4 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-health.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtHealthTool = Tool.define("dbt_health", { + description: "DataPilot project health insights for the dbt project.", + parameters: z.object({ + config_path: z.string().optional().describe("DataPilot config file to use"), + }), + async execute() { + return { + title: "dbt_health: not implemented", + metadata: { success: false }, + output: "dbt_health is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-map-relations.ts b/packages/opencode/src/altimate/tools/dbt-map-relations.ts new file mode 100644 index 0000000000..1216d6b2ee --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-map-relations.ts @@ -0,0 +1,18 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtMapRelationsTool = Tool.define("dbt_map_relations", { + description: "Map each table a raw SQL query reads to the dbt ref() or source() for it, from the manifest.", + parameters: z.object({ + sql: z.string().describe("Raw SQL to map"), + dialect: z.string().optional().describe("SQL dialect of the query"), + manifest_path: z.string().optional().describe("manifest.json to match against (default: the dbt project's target/manifest.json)"), + }), + async execute() { + return { + title: "dbt_map_relations: not implemented", + metadata: { success: false }, + output: "dbt_map_relations is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-run-results.ts b/packages/opencode/src/altimate/tools/dbt-run-results.ts new file mode 100644 index 0000000000..86e68a1668 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-run-results.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtRunResultsTool = Tool.define("dbt_run_results", { + description: "The latest dbt run's results from target/run_results.json: command, duration and each node's status.", + parameters: z.object({ + project_dir: z.string().optional().describe("dbt project folder (default: the dbt project found from the working directory)"), + }), + async execute() { + return { + title: "dbt_run_results: not implemented", + metadata: { success: false }, + output: "dbt_run_results is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts new file mode 100644 index 0000000000..b2af1b0f18 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtSqlGraphTool = Tool.define("dbt_sql_graph", { + description: "A dbt model's SQL as a graph: its CTEs, joins and filters, and the edges between them.", + parameters: z.object({ + model: z.string().describe("Model name"), + }), + async execute() { + return { + title: "dbt_sql_graph: not implemented", + metadata: { success: false }, + output: "dbt_sql_graph is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/docs-share-comments.ts b/packages/opencode/src/altimate/tools/docs-share-comments.ts new file mode 100644 index 0000000000..2c8c0a2ab3 --- /dev/null +++ b/packages/opencode/src/altimate/tools/docs-share-comments.ts @@ -0,0 +1,20 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DocsShareCommentsTool = Tool.define("docs_share_comments", { + description: "Comment threads on shared dbt docs: list open threads, reply to one, or resolve one.", + parameters: z.object({ + action: z.enum(["list", "reply", "resolve"]).describe("What to do"), + project: z.string().optional().describe("dbt project name the shared docs belong to"), + share_id: z.number().int().optional().describe("Shared docs id"), + group_id: z.number().int().optional().describe("Comment thread id"), + text: z.string().optional().describe("Reply text"), + }), + async execute() { + return { + title: "docs_share_comments: not implemented", + metadata: { success: false }, + output: "docs_share_comments is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts new file mode 100644 index 0000000000..b565e5a58a --- /dev/null +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -0,0 +1,221 @@ +// The tools the dbt Power User features call: their allow-list entries and placeholder answers, the +// one gate every dbt-adapter-backed tool goes through, and the API helper for new endpoint files. The +// dbt adapter is replaced through `dbtAdapterInternals`, so these tests need no Python or dbt. +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { realpathSync } from "node:fs" +import os from "node:os" +import path from "node:path" +import { Effect } from "effect" +import type { Tool } from "@/tool/tool" +import { SessionID, MessageID } from "@/session/schema" +import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" +import { dbtAdapterAccessInternals, dbtAdapterFor } from "@/altimate/tools/dbt-adapter-access" +import { EXPOSED, MCP_OPEN_WRITE_TOOLS, WRITE_TOOLS } from "@/altimate/tool-cli/allow-list" +import { AltimateApi } from "@/altimate/api/client" +import { Instance } from "@/project/instance" +import { disposeInstance } from "@/effect/instance-registry" +import { initTool } from "./tool-fixture" + +/** Each tool with the smallest arguments its parameters accept. */ +const MINIMAL_ARGS: Record> = { + dbt_run_results: {}, + dbt_defer_state: { project_dir: "." }, + dbt_map_relations: { sql: "select 1" }, + docs_share_comments: { action: "list" }, + dbt_cost_estimate: {}, + dbt_health: {}, + dbt_sql_graph: { model: "customers" }, + dbt_docs_coverage: { model: "customers" }, + dbt_cte_profile: { model: "customers" }, +} +const FEATURE_TOOLS = Object.keys(MINIMAL_ARGS) + +const ctx: Tool.Context = { + sessionID: SessionID.make("ses_dbt_adapter_access"), + messageID: MessageID.make("msg_dbt_adapter_access"), + agent: "analyst", + abort: new AbortController().signal, + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, +} + +let project: string + +function useConfig() { + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: project, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) +} + +beforeAll(async () => { + project = await mkdtemp(path.join(os.tmpdir(), "dbt-adapter-access-")) + await writeFile(path.join(project, "dbt_project.yml"), "name: jaffle_shop\n") +}) + +afterAll(async () => { + await rm(project, { recursive: true, force: true }) +}) + +beforeEach(() => resetDbtAdapter()) + +afterEach(() => { + resetDbtAdapter() + delete dbtAdapterInternals.readConfig + delete dbtAdapterInternals.create +}) + +describe("feature tools in the allow-list", () => { + test("all nine are exposed, in the map's alphabetical order, and none is a write tool", () => { + const ids = Object.keys(EXPOSED) + for (const id of FEATURE_TOOLS) { + expect(ids).toContain(id) + expect(WRITE_TOOLS.has(id)).toBe(false) + expect(MCP_OPEN_WRITE_TOOLS.has(id)).toBe(false) + } + const listed = ids.filter((id) => id.startsWith("dbt_") || id.startsWith("docs_")) + expect(listed).toEqual([...listed].sort()) + }) + + test("each placeholder answers that it isn't implemented, as a failure", async () => { + for (const id of FEATURE_TOOLS) { + const tool = await initTool(EXPOSED[id] as any) + expect(tool.id).toBe(id) + const result = await tool.execute(MINIMAL_ARGS[id], ctx) + expect(result.title).toBe(`${id}: not implemented`) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain(id) + } + }) +}) + +describe("dbtAdapterFor", () => { + let created: number + beforeEach(() => { + created = 0 + useConfig() + dbtAdapterInternals.create = async () => { + created += 1 + return { isInitialized: () => true, dispose: async () => {} } + } + }) + + test("refuses a read-only call and never creates the adapter", async () => { + const access = await dbtAdapterFor({ ...ctx, extra: { forceReadOnly: true } }) + expect(access.adapter).toBeUndefined() + expect(access.refusal).toBe("read_only") + expect(access.reason).toContain("--allow-write") + expect(created).toBe(0) + }) + + test("refuses a plan-mode call", async () => { + const access = await dbtAdapterFor({ ...ctx, agent: "plan" }) + expect(access.refusal).toBe("read_only") + expect(created).toBe(0) + }) + + test("returns the adapter where writes are allowed", async () => { + const access = await dbtAdapterFor(ctx) + expect(access.refusal).toBeUndefined() + expect(access.adapter?.isInitialized()).toBe(true) + expect(created).toBe(1) + }) + + test("says when the adapter didn't load the project", async () => { + dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_loaded", reason: "the dbt adapter didn't load the project" }) + }) + + test("says there is no dbt project when the config finds none", async () => { + dbtAdapterInternals.readConfig = async () => null + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "no_project", reason: "no dbt project here" }) + }) + + test("says when the adapter doesn't start in time", async () => { + dbtAdapterInternals.create = () => new Promise(() => {}) + const saved = dbtAdapterAccessInternals.adapterStartTimeoutMs + dbtAdapterAccessInternals.adapterStartTimeoutMs = 20 + try { + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start within 0.02 s" }) + } finally { + dbtAdapterAccessInternals.adapterStartTimeoutMs = saved + } + }) + + test("a cancelled call throws rather than answering", async () => { + const controller = new AbortController() + controller.abort() + await expect(dbtAdapterFor({ ...ctx, abort: controller.signal })).rejects.toThrow() + expect(created).toBe(0) + }) + + test("a session in a dbt project the adapter isn't set up for says so", async () => { + dbtAdapterInternals.readConfig = async () => null + try { + const access = await Instance.provide({ directory: project, fn: () => dbtAdapterFor(ctx) }) + expect(access).toEqual({ + refusal: "other_project", + reason: `the dbt adapter isn't set up for ${realpathSync(project)}`, + }) + expect(created).toBe(0) + } finally { + await disposeInstance(project) + } + }) +}) + +describe("AltimateApi.apiRequest", () => { + let home: string + let saved: string | undefined + beforeEach(async () => { + home = await mkdtemp(path.join(os.tmpdir(), "altimate-api-home-")) + saved = process.env.OPENCODE_TEST_HOME + process.env.OPENCODE_TEST_HOME = home + }) + afterEach(async () => { + if (saved === undefined) delete process.env.OPENCODE_TEST_HOME + else process.env.OPENCODE_TEST_HOME = saved + await rm(home, { recursive: true, force: true }) + }) + + test("fails with a clear message when no credentials are stored", async () => { + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow( + `No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${path.join(home, ".altimate", "altimate.json")}).`, + ) + }) + + test("sends the stored key and tenant and returns the JSON body", async () => { + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: "https://api.example.test/", altimateInstanceName: "acme", altimateApiKey: "k1" }), + ) + const original = globalThis.fetch + const seen: { url: string; init: RequestInit }[] = [] + globalThis.fetch = (async (url: string, init: RequestInit) => { + seen.push({ url, init }) + return new Response(JSON.stringify({ ok: 1 }), { status: 200 }) + }) as unknown as typeof fetch + try { + expect(await AltimateApi.apiRequest("POST", "/dbt/v1/x", { a: 1 })).toEqual({ ok: 1 }) + expect(seen[0].url).toBe("https://api.example.test/dbt/v1/x") + const headers = seen[0].init.headers as Record + expect(headers.Authorization).toBe("Bearer k1") + expect(headers["x-tenant"]).toBe("acme") + expect(seen[0].init.body).toBe(JSON.stringify({ a: 1 })) + } finally { + globalThis.fetch = original + } + }) + + test("names the file when the stored credentials can't be read", async () => { + await Bun.write(path.join(home, ".altimate", "altimate.json"), "{not json") + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow("can't be used") + }) +}) From 21d0494cfbb1c87a2b1b3ba6e347f4f1be149856 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 17:01:49 +0530 Subject: [PATCH 2/8] fix: harden the dbt Power User tool foundation Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/api/client.ts | 4 +- .../src/altimate/tools/dbt-adapter-access.ts | 3 +- .../src/altimate/tools/dbt-cost-estimate.ts | 2 +- .../src/altimate/tools/dbt-cte-profile.ts | 2 +- .../src/altimate/tools/dbt-defer-state.ts | 2 +- .../src/altimate/tools/dbt-docs-coverage.ts | 10 ++-- .../opencode/src/altimate/tools/dbt-health.ts | 2 +- .../src/altimate/tools/dbt-map-relations.ts | 7 ++- .../src/altimate/tools/dbt-run-results.ts | 7 ++- .../src/altimate/tools/dbt-sql-graph.ts | 2 +- .../src/altimate/tools/docs-share-comments.ts | 2 +- .../test/altimate/dbt-adapter-access.test.ts | 52 ++++++++++++++++++- 12 files changed, 78 insertions(+), 17 deletions(-) diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index 069a182b7a..cb53c0ec7b 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -359,7 +359,9 @@ export namespace AltimateApi { } catch (e) { const p = credentialsPath() if (!(await Filesystem.exists(p))) - throw new Error(`No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${p}).`) + throw new Error( + `No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${p}).`, + ) throw new Error(`The Altimate credentials in ${p} can't be used: ${e instanceof Error ? e.message : String(e)}`) } return request(creds, method, endpoint, body) diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index a2711e35d1..d73dd7e9c5 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -67,7 +67,8 @@ export async function dbtAdapterFor(ctx: Pick { expect(tool.id).toBe(id) const result = await tool.execute(MINIMAL_ARGS[id], ctx) expect(result.title).toBe(`${id}: not implemented`) - expect(result.metadata.success).toBe(false) + expect(result.metadata).toEqual({ success: false, error: "not implemented", truncated: false }) expect(result.output).toContain(id) } }) @@ -149,6 +149,30 @@ describe("dbtAdapterFor", () => { } }) + test("says the adapter didn't start when the project exists but creating it fails", async () => { + dbtAdapterInternals.create = async () => { + created += 1 + throw new Error("dbt-integration missing") + } + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start" }) + expect(created).toBe(1) + }) + + test("a call cancelled while the adapter starts throws and stops waiting", async () => { + let entered = () => {} + const starting = new Promise((resolve) => (entered = resolve)) + dbtAdapterInternals.create = () => { + entered() + return new Promise(() => {}) + } + const controller = new AbortController() + const pending = dbtAdapterFor({ ...ctx, abort: controller.signal }) + await starting + controller.abort(new Error("cancelled")) + await expect(pending).rejects.toThrow("cancelled") + }) + test("a cancelled call throws rather than answering", async () => { const controller = new AbortController() controller.abort() @@ -169,6 +193,28 @@ describe("dbtAdapterFor", () => { await disposeInstance(project) } }) + + test("a session in another dbt project than the adapter's says which", async () => { + const other = await mkdtemp(path.join(os.tmpdir(), "dbt-adapter-other-")) + await writeFile(path.join(other, "dbt_project.yml"), "name: other\n") + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: other, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) + try { + const access = await Instance.provide({ directory: project, fn: () => dbtAdapterFor(ctx) }) + expect(access).toEqual({ + refusal: "other_project", + reason: `the dbt adapter is set up for ${realpathSync(other)}, not ${realpathSync(project)}`, + }) + expect(created).toBe(0) + } finally { + await disposeInstance(project) + await rm(other, { recursive: true, force: true }) + } + }) }) describe("AltimateApi.apiRequest", () => { @@ -216,6 +262,8 @@ describe("AltimateApi.apiRequest", () => { test("names the file when the stored credentials can't be read", async () => { await Bun.write(path.join(home, ".altimate", "altimate.json"), "{not json") - await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow("can't be used") + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow( + `The Altimate credentials in ${path.join(home, ".altimate", "altimate.json")} can't be used`, + ) }) }) From 5971ad2bfd2ba370a03a539c4c7b645295dd7207 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 17:08:32 +0530 Subject: [PATCH 3/8] feat: dbt_map_relations maps the tables of raw SQL to ref() and source() calls Each table the SQL reads is matched against the relation_name of the manifest's models, snapshots, seeds and sources, with the match reported as exact, case_insensitive, partial or none and every candidate listed. Co-Authored-By: Claude Sonnet 5.5 --- .../src/altimate/tools/dbt-map-relations.ts | 449 +++++++++++++++++- .../test/altimate/dbt-map-relations.test.ts | 339 +++++++++++++ 2 files changed, 779 insertions(+), 9 deletions(-) create mode 100644 packages/opencode/test/altimate/dbt-map-relations.test.ts diff --git a/packages/opencode/src/altimate/tools/dbt-map-relations.ts b/packages/opencode/src/altimate/tools/dbt-map-relations.ts index 1216d6b2ee..5036de5852 100644 --- a/packages/opencode/src/altimate/tools/dbt-map-relations.ts +++ b/packages/opencode/src/altimate/tools/dbt-map-relations.ts @@ -1,18 +1,449 @@ import z from "zod" +import fs from "fs" +import path from "path" import { Tool } from "../../tool/tool" +import { Dispatcher } from "../native" +import { loadRawManifest, detectDialect } from "../native/dbt/helpers" +import { projectRoot } from "../native/connections/registry" + +export type RelationMatch = "exact" | "case_insensitive" | "partial" | "none" +export type RelationKind = "model" | "snapshot" | "seed" | "source" + +export type RelationCandidate = { + replaceWith: string + kind: RelationKind + name: string + uniqueId: string + relationName: string + match: Exclude +} + +export type RelationMapping = { + /** The table as written in the SQL, qualifiers included. */ + raw: string + /** A `ref()` or `source()` call, or `raw` itself when nothing matched. */ + replaceWith: string + match: RelationMatch + /** Every manifest relation the table matched, best first. */ + candidates: RelationCandidate[] +} + +type Relation = { + kind: RelationKind + name: string + uniqueId: string + relationName: string + parts: string[] + call: string +} + +const RANK: Record, number> = { exact: 0, case_insensitive: 1, partial: 2 } +const KIND_RANK: Record = { model: 0, snapshot: 1, seed: 2, source: 3 } + +/** + * Split a dotted name into its parts, honouring "double", `back` and [bracket] quotes. Dots inside + * backticks split the name when `splitBacktickDots` is set: BigQuery reads `proj.ds.tbl` as a path. + */ +export function splitQualified(text: string, splitBacktickDots = false): string[] { + const parts: string[] = [] + let cur = "" + let i = 0 + const push = () => { + parts.push(cur) + cur = "" + } + while (i < text.length) { + const ch = text[i] + if (ch === '"' || ch === "`" || ch === "[") { + const close = ch === "[" ? "]" : ch + let j = i + 1 + let body = "" + while (j < text.length) { + if (text[j] === close) { + if (text[j + 1] === close && close !== "]") { + body += close + j += 2 + continue + } + break + } + body += text[j++] + } + if (ch === "`" && splitBacktickDots) { + const sub = body.split(".") + cur += sub[0] + for (const s of sub.slice(1)) { + push() + cur = s + } + } else cur += body + i = j + 1 + } else if (ch === ".") { + push() + i++ + } else { + cur += ch + i++ + } + } + push() + return parts +} + +/** A table reference found in the SQL text: the text as written and its unquoted parts. */ +type Reference = { raw: string; parts: string[] } + +const IDENT_START = /[A-Za-z_$"`[]/ +const RESERVED = new Set([ + "select", + "from", + "where", + "join", + "inner", + "left", + "right", + "full", + "outer", + "cross", + "natural", + "lateral", + "on", + "using", + "group", + "order", + "having", + "limit", + "offset", + "union", + "intersect", + "except", + "minus", + "qualify", + "window", + "set", + "when", + "then", + "else", + "end", + "as", + "values", + "returning", + "fetch", + "for", + "pivot", + "unpivot", + "tablesample", + "inner", + "straight_join", + "asof", +]) + +/** One identifier chain (`a.b.c`, each part quoted or not) starting at `i`; null when there is none. */ +function readChain(sql: string, start: number): { raw: string; end: number } | null { + let i = start + const n = sql.length + while (i < n) { + const ch = sql[i] + if (ch === '"' || ch === "`" || ch === "[") { + const close = ch === "[" ? "]" : ch + i++ + while (i < n) { + if (sql[i] === close) { + if (sql[i + 1] === close && close !== "]") { + i += 2 + continue + } + break + } + i++ + } + if (i >= n) return null + i++ + } else if (/[A-Za-z0-9_$\-]/.test(ch) && (i > start || /[A-Za-z_$]/.test(ch))) { + while (i < n && /[A-Za-z0-9_$]/.test(sql[i])) i++ + // BigQuery project ids may contain hyphens (`my-proj.ds.tbl` unquoted). + while (sql[i] === "-" && /[A-Za-z0-9_]/.test(sql[i + 1] ?? "")) { + i++ + while (i < n && /[A-Za-z0-9_$]/.test(sql[i])) i++ + } + } else return null + if (sql[i] === ".") { + i++ + continue + } + break + } + return i > start ? { raw: sql.slice(start, i), end: i } : null +} + +/** Replace comments and string literals with spaces so the scan never reads inside them. */ +function blank(sql: string): string { + let out = "" + let i = 0 + while (i < sql.length) { + const two = sql.slice(i, i + 2) + if (two === "--") { + while (i < sql.length && sql[i] !== "\n") { + out += " " + i++ + } + } else if (two === "/*") { + const end = sql.indexOf("*/", i + 2) + const stop = end < 0 ? sql.length : end + 2 + out += sql.slice(i, stop).replace(/[^\n]/g, " ") + i = stop + } else if (sql[i] === "'") { + let j = i + 1 + while (j < sql.length) { + if (sql[j] === "\\") j += 2 + else if (sql[j] === "'") { + if (sql[j + 1] === "'") j += 2 + else break + } else j++ + } + const stop = Math.min(j + 1, sql.length) + out += sql.slice(i, stop).replace(/[^\n]/g, " ") + i = stop + } else if (sql[i] === '"' || sql[i] === "`") { + // Keep quoted identifiers intact; skip over them so a quote inside one is not misread. + const close = sql[i] + let j = i + 1 + while (j < sql.length && sql[j] !== close) j++ + out += sql.slice(i, Math.min(j + 1, sql.length)) + i = Math.min(j + 1, sql.length) + } else { + out += sql[i] + i++ + } + } + return out +} + +function skipSpace(sql: string, i: number): number { + while (i < sql.length && /\s/.test(sql[i])) i++ + return i +} + +/** + * The table references of FROM, JOIN, INTO and UPDATE clauses, in text order, written as in the SQL. + * altimate-core's table list keeps only the last name part, so the schema and database a person wrote + * are read back from the text here; the list decides which of these are real tables. + */ +export function scanTableReferences(sql: string, splitBacktickDots: boolean): Reference[] { + const text = blank(sql) + const found: Reference[] = [] + const re = /\b(from|join|into|update|using|table)\b/gi + let m: RegExpExecArray | null + while ((m = re.exec(text))) { + let i = skipSpace(text, m.index + m[0].length) + for (;;) { + if (!IDENT_START.test(text[i] ?? "")) break + const chain = readChain(text, i) + if (!chain) break + i = skipSpace(text, chain.end) + // A function call (`flatten(...)`, `unnest(...)`) is not a table. + if (text[i] === "(") break + if (!RESERVED.has(chain.raw.toLowerCase())) { + found.push({ raw: chain.raw, parts: splitQualified(chain.raw, splitBacktickDots) }) + } else break + // An alias, with or without AS, then a comma continues a FROM list. + const asM = /^as\s+/i.exec(text.slice(i)) + if (asM) i = skipSpace(text, i + asM[0].length) + const alias = readChain(text, i) + if (alias && !alias.raw.includes(".") && !RESERVED.has(alias.raw.toLowerCase())) i = skipSpace(text, alias.end) + else if (asM && alias) i = skipSpace(text, alias.end) + if (text[i] === ",") { + i = skipSpace(text, i + 1) + continue + } + break + } + } + return found +} + +/** The model, snapshot, seed and source nodes of a manifest, each with its relation's name parts. */ +export function manifestRelations(manifest: any): Relation[] { + const rootProject: string | undefined = manifest?.metadata?.project_name + const out: Relation[] = [] + const relationOf = (node: any): string | undefined => { + if (typeof node?.relation_name === "string" && node.relation_name) return node.relation_name + const db = node?.database + const schema = node?.schema + const name = node?.alias ?? node?.identifier ?? node?.name + if (typeof schema === "string" && schema && typeof name === "string" && name) { + return [db, schema, name].filter((p) => typeof p === "string" && p).join(".") + } + return undefined + } + const nodes: Record = manifest?.nodes ?? {} + for (const [uniqueId, node] of Object.entries(nodes)) { + const kind = node?.resource_type + if (kind !== "model" && kind !== "snapshot" && kind !== "seed") continue + // Ephemeral models are inlined by dbt and have no relation to match. + if (node?.config?.materialized === "ephemeral") continue + const relationName = relationOf(node) + if (!relationName || typeof node.name !== "string") continue + const pkg = + typeof node.package_name === "string" && rootProject && node.package_name !== rootProject + ? node.package_name + : undefined + const call = pkg ? `{{ ref('${pkg}', '${node.name}') }}` : `{{ ref('${node.name}') }}` + out.push({ kind, name: node.name, uniqueId, relationName, parts: splitQualified(relationName, true), call }) + } + const sources: Record = manifest?.sources ?? {} + for (const [uniqueId, node] of Object.entries(sources)) { + const relationName = relationOf(node) + if (!relationName || typeof node?.source_name !== "string" || typeof node?.name !== "string") continue + out.push({ + kind: "source", + name: node.name, + uniqueId, + relationName, + parts: splitQualified(relationName, true), + call: `{{ source('${node.source_name}', '${node.name}') }}`, + }) + } + return out +} + +/** How a table written as `parts` matches a relation, or null when it does not. */ +export function matchParts(parts: string[], relation: string[]): Exclude | null { + if (parts.length === 0 || parts.length > relation.length) return null + const tail = relation.slice(relation.length - parts.length) + const same = parts.every((p, i) => p === tail[i]) + const fold = parts.every((p, i) => p.toLowerCase() === tail[i].toLowerCase()) + if (parts.length < relation.length) return fold ? "partial" : null + return same ? "exact" : fold ? "case_insensitive" : null +} + +/** Map each table of `sql` to the manifest's ref() or source() call that builds it. */ +export function mapRelations(tables: string[], sql: string, relations: Relation[], dialect: string): RelationMapping[] { + const splitDots = dialect === "bigquery" + const refs = scanTableReferences(sql, splitDots) + // Reference list keyed by the written form, in text order; only tables altimate-core reported count. + const wanted = new Set(tables) + const entries: Reference[] = [] + const seen = new Set() + const add = (r: Reference) => { + const key = r.parts.join("\u0000") + if (seen.has(key)) return + seen.add(key) + entries.push(r) + } + const covered = new Set() + for (const r of refs) { + const last = r.parts[r.parts.length - 1] + const whole = r.parts.join(".") + if (wanted.has(last) || wanted.has(whole)) { + add(r) + covered.add(wanted.has(last) ? last : whole) + } + } + for (const t of tables) { + if (covered.has(t)) continue + add({ raw: t, parts: splitQualified(t, splitDots) }) + } + return entries.map((entry) => { + const hits: RelationCandidate[] = [] + for (const rel of relations) { + const how = matchParts(entry.parts, rel.parts) + if (!how) continue + hits.push({ + replaceWith: rel.call, + kind: rel.kind, + name: rel.name, + uniqueId: rel.uniqueId, + relationName: rel.relationName, + match: how, + }) + } + hits.sort( + (a, b) => + RANK[a.match] - RANK[b.match] || KIND_RANK[a.kind] - KIND_RANK[b.kind] || a.uniqueId.localeCompare(b.uniqueId), + ) + const best = hits[0] + return { + raw: entry.raw, + replaceWith: best ? best.replaceWith : entry.raw, + match: best ? best.match : "none", + candidates: hits, + } + }) +} + +export function formatMapping(rows: RelationMapping[]): string { + if (rows.length === 0) return "No tables found in the SQL." + const order: RelationMatch[] = ["none", "partial", "case_insensitive", "exact"] + const lines: string[] = [] + for (const match of order) { + for (const r of rows.filter((x) => x.match === match)) { + const more = r.candidates.length > 1 ? ` (${r.candidates.length} candidates)` : "" + lines.push( + match === "none" ? `none: ${r.raw} -> no model or source` : `${match}: ${r.raw} -> ${r.replaceWith}${more}`, + ) + } + } + return lines.join("\n") +} + +type MapRelationsMetadata = { + success: boolean + mappings: RelationMapping[] + error?: string + dialect?: string + manifest_path?: string +} export const DbtMapRelationsTool = Tool.define("dbt_map_relations", { - description: "Map each table a raw SQL query reads to the dbt ref() or source() for it, from the manifest.", + description: + "Map the tables in a SQL query to dbt ref() and source() calls, from the project's manifest: each table is matched to the relation_name of a model, snapshot, seed or source. Returns, per table, the replacement, how it matched (exact, case_insensitive, partial, none) and the candidates. Read-only.", parameters: z.object({ - sql: z.string().describe("Raw SQL to map"), - dialect: z.string().optional().describe("SQL dialect of the query"), - manifest_path: z.string().optional().describe("manifest.json to match against (default: the dbt project's target/manifest.json)"), + sql: z.string().describe("The raw SQL to map"), + dialect: z.string().optional().describe("SQL dialect (default: the manifest's adapter)"), + manifest_path: z + .string() + .optional() + .describe("manifest.json to match against (default: the dbt project's target/manifest.json)"), }), - async execute() { - return { - title: "dbt_map_relations: not implemented", - metadata: { success: false }, - output: "dbt_map_relations is not implemented in this build of altimate-code yet.", + async execute(args): Promise<{ title: string; metadata: MapRelationsMetadata; output: string }> { + const fail = (msg: string) => ({ + title: "dbt Map Relations: ERROR", + metadata: { success: false, error: msg, mappings: [] } as MapRelationsMetadata, + output: msg, + }) + try { + const manifestPath = args.manifest_path ?? (await defaultManifestPath()) + const manifest = loadRawManifest(manifestPath) + if (!manifest) return fail(`No manifest at ${manifestPath}. Run \`dbt parse\` or pass manifest_path.`) + const dialect = args.dialect || detectDialect(manifest) + const result = await Dispatcher.call("altimate_core.metadata", { sql: args.sql, dialect }) + const data = (result.data ?? {}) as Record + const error = result.error ?? data.error + if (!result.success || error) return fail(`Could not read the tables of the SQL: ${error ?? "parse failed"}`) + const tables: string[] = Array.isArray(data.tables) + ? data.tables.filter((t: unknown) => typeof t === "string") + : [] + const mappings = mapRelations(tables, args.sql, manifestRelations(manifest), dialect) + const count = (m: RelationMatch) => mappings.filter((r) => r.match === m).length + return { + title: `dbt Map Relations: ${mappings.length} tables, ${count("none")} unmatched`, + metadata: { success: true, dialect, manifest_path: manifestPath, mappings }, + output: formatMapping(mappings), + } + } catch (e) { + return fail(e instanceof Error ? e.message : String(e)) } }, }) + +async function defaultManifestPath(): Promise { + const { findProjectRoot } = await import("../../../../dbt-tools/src/config") + const found = findProjectRoot(projectRoot()) + const root = found ?? (await (await import("../native/connections/register")).getDbtProjectRoot()) + let real = root ?? "" + try { + real = root ? fs.realpathSync(root) : "" + } catch {} + return path.join(real, "target", "manifest.json") +} diff --git a/packages/opencode/test/altimate/dbt-map-relations.test.ts b/packages/opencode/test/altimate/dbt-map-relations.test.ts new file mode 100644 index 0000000000..f568090e77 --- /dev/null +++ b/packages/opencode/test/altimate/dbt-map-relations.test.ts @@ -0,0 +1,339 @@ +/** + * dbt_map_relations: maps the tables of a raw SQL query to ref() and source() calls. + * + * The first block pins what altimate-core's extractMetadata returns for quoted, upper-case and + * qualified names: it keeps only the last name part (quotes removed, case kept) and merges tables + * that share it, so the matching rules read the qualifiers back from the SQL text. + */ +import { describe, test, expect, afterEach } from "bun:test" +import { mkdtempSync, rmSync, writeFileSync } from "fs" +import { tmpdir } from "os" +import { join } from "path" +import { Effect } from "effect" +import type { Tool } from "@/tool/tool" +import { SessionID, MessageID } from "@/session/schema" +import { Dispatcher } from "../../src/altimate/native" +import { initTool } from "./tool-fixture" +import { + DbtMapRelationsTool, + manifestRelations, + mapRelations, + matchParts, + scanTableReferences, + splitQualified, +} from "../../src/altimate/tools/dbt-map-relations" + +async function tablesOf(sql: string, dialect: string): Promise { + const result = await Dispatcher.call("altimate_core.metadata", { sql, dialect }) + expect(result.success).toBe(true) + return ((result.data ?? {}) as { tables?: string[] }).tables ?? [] +} + +describe("altimate-core extractMetadata table names", () => { + test("Snowflake: last name part only, quotes removed, case kept, one entry per bare name", async () => { + expect( + await tablesOf( + 'select * from RAW.JAFFLE.Customers c join "raw"."jaffle"."orders" o on 1=1 join analytics.stg_x x on 1=1', + "snowflake", + ), + ).toEqual(["Customers", "orders", "stg_x"]) + expect(await tablesOf("select * from db.sch.orders o join other.orders o2 on 1=1", "snowflake")).toEqual(["orders"]) + expect( + await tablesOf("select * from RAW.JAFFLE.CUSTOMERS a join raw.jaffle.customers b on 1=1", "snowflake"), + ).toEqual(["CUSTOMERS", "customers"]) + expect(await tablesOf('select * from "My Schema"."My Table" t', "snowflake")).toEqual(["My Table"]) + }) + + test("BigQuery: a single backtick path comes back whole, a split one by its last part", async () => { + expect( + await tablesOf( + "select * from `proj.ds.Customers` a join `proj`.`ds`.`Orders` b on 1=1 join ds.x c on 1=1", + "bigquery", + ), + ).toEqual(["Orders", "proj.ds.Customers", "x"]) + expect(await tablesOf("select * from proj-1.ds.Customers a", "bigquery")).toEqual(["Customers"]) + }) + + test("Databricks: three-part and backticked names reduce to the last part", async () => { + expect( + await tablesOf( + "select * from main.raw.Customers a join raw.orders b on 1=1 join `main`.`raw`.`items` c on 1=1", + "databricks", + ), + ).toEqual(["Customers", "items", "orders"]) + }) + + test("CTE names are not tables", async () => { + expect(await tablesOf("with a as (select * from raw.customers) select * from a", "snowflake")).toEqual([ + "customers", + ]) + }) +}) + +describe("splitQualified", () => { + test("splits on dots outside quotes", () => { + expect(splitQualified('"RAW"."JAFFLE"."CUSTOMERS"')).toEqual(["RAW", "JAFFLE", "CUSTOMERS"]) + expect(splitQualified('"a.b"."c"')).toEqual(["a.b", "c"]) + expect(splitQualified("`proj`.`ds`.`t`")).toEqual(["proj", "ds", "t"]) + expect(splitQualified("[db].[dbo].[t]")).toEqual(["db", "dbo", "t"]) + }) + test("backtick dots split only when asked", () => { + expect(splitQualified("`proj.ds.t`", true)).toEqual(["proj", "ds", "t"]) + expect(splitQualified("`proj.ds.t`")).toEqual(["proj.ds.t"]) + }) +}) + +describe("scanTableReferences", () => { + test("reads qualified names from FROM, JOIN and comma lists, written as in the SQL", () => { + const refs = scanTableReferences( + 'select * from raw.jaffle.customers c, "RAW"."JAFFLE"."orders" as o left join other.payments p on 1=1', + false, + ) + expect(refs.map((r) => r.raw)).toEqual(["raw.jaffle.customers", '"RAW"."JAFFLE"."orders"', "other.payments"]) + expect(refs[1].parts).toEqual(["RAW", "JAFFLE", "orders"]) + }) + test("ignores table functions, subqueries, comments and strings", () => { + const refs = scanTableReferences( + "select 'from fake.t' as s from table(flatten(x)) f join (select 1) q on 1=1 /* from c.d */ -- from e.f\n join real.t t on 1=1", + false, + ) + expect(refs.map((r) => r.raw)).toEqual(["real.t"]) + }) + test("reads hyphenated BigQuery project ids", () => { + expect(scanTableReferences("select * from my-proj.ds.tbl", true)[0].parts).toEqual(["my-proj", "ds", "tbl"]) + }) +}) + +describe("matchParts", () => { + const rel = ["RAW", "JAFFLE", "CUSTOMERS"] + test("exact, case_insensitive, partial and no match", () => { + expect(matchParts(["RAW", "JAFFLE", "CUSTOMERS"], rel)).toBe("exact") + expect(matchParts(["raw", "jaffle", "customers"], rel)).toBe("case_insensitive") + expect(matchParts(["jaffle", "customers"], rel)).toBe("partial") + expect(matchParts(["customers"], rel)).toBe("partial") + expect(matchParts(["other", "jaffle", "customers"], rel)).toBeNull() + expect(matchParts(["x", "raw", "jaffle", "customers"], rel)).toBeNull() + }) +}) + +const MANIFEST = { + metadata: { adapter_type: "snowflake", project_name: "shop" }, + nodes: { + "model.shop.customers": { + resource_type: "model", + name: "customers", + package_name: "shop", + relation_name: '"ANALYTICS"."MARTS"."CUSTOMERS"', + config: { materialized: "table" }, + }, + "model.shop.stg_orders": { + resource_type: "model", + name: "stg_orders", + package_name: "shop", + relation_name: '"ANALYTICS"."STAGING"."STG_ORDERS"', + config: { materialized: "view" }, + }, + "model.shop.inlined": { + resource_type: "model", + name: "inlined", + package_name: "shop", + relation_name: null, + config: { materialized: "ephemeral" }, + }, + "model.utils.date_spine": { + resource_type: "model", + name: "date_spine", + package_name: "utils", + relation_name: '"ANALYTICS"."UTILS"."DATE_SPINE"', + config: { materialized: "table" }, + }, + "seed.shop.countries": { + resource_type: "seed", + name: "countries", + package_name: "shop", + relation_name: '"ANALYTICS"."SEEDS"."COUNTRIES"', + }, + "test.shop.not_null": { resource_type: "test", name: "not_null", relation_name: '"X"."Y"."Z"' }, + }, + sources: { + "source.shop.jaffle.customers": { + resource_type: "source", + source_name: "jaffle", + name: "customers", + relation_name: '"RAW"."JAFFLE"."CUSTOMERS"', + }, + "source.shop.jaffle.payments": { + resource_type: "source", + source_name: "jaffle", + name: "payments", + relation_name: '"RAW"."JAFFLE"."PAYMENTS"', + }, + }, +} + +describe("mapRelations", () => { + const relations = manifestRelations(MANIFEST) + const map = (sql: string, tables: string[], dialect = "snowflake") => mapRelations(tables, sql, relations, dialect) + + test("manifestRelations skips ephemeral models and non-relations, and names other packages' models", () => { + const names = relations.map((r) => r.uniqueId).sort() + expect(names).not.toContain("model.shop.inlined") + expect(names).not.toContain("test.shop.not_null") + expect(relations.find((r) => r.name === "date_spine")?.call).toBe("{{ ref('utils', 'date_spine') }}") + }) + + test("exact: every part written, same case", () => { + const [row] = map('select * from "RAW"."JAFFLE"."PAYMENTS"', ["PAYMENTS"]) + expect(row.match).toBe("exact") + expect(row.replaceWith).toBe("{{ source('jaffle', 'payments') }}") + expect(row.raw).toBe('"RAW"."JAFFLE"."PAYMENTS"') + expect(row.candidates).toHaveLength(1) + }) + + test("case_insensitive: every part written, different case", () => { + const [row] = map("select * from raw.jaffle.payments", ["payments"]) + expect(row.match).toBe("case_insensitive") + expect(row.replaceWith).toBe("{{ source('jaffle', 'payments') }}") + }) + + test("partial: a missing database or schema, a model preferred over a source of the same name", () => { + const [row] = map("select * from customers", ["customers"]) + expect(row.match).toBe("partial") + expect(row.replaceWith).toBe("{{ ref('customers') }}") + expect(row.candidates.map((c) => c.kind)).toEqual(["model", "source"]) + }) + + test("a written schema narrows the match to one relation", () => { + const [row] = map("select * from jaffle.customers", ["customers"]) + expect(row.match).toBe("partial") + expect(row.replaceWith).toBe("{{ source('jaffle', 'customers') }}") + expect(row.candidates).toHaveLength(1) + }) + + test("a fully written name picks the source, not the model that shares its last part", () => { + const [row] = map("select * from raw.jaffle.customers", ["customers"]) + expect(row.match).toBe("case_insensitive") + expect(row.replaceWith).toBe("{{ source('jaffle', 'customers') }}") + }) + + test("a better match beats a preferred kind", () => { + const [row] = map('select * from "RAW"."JAFFLE"."CUSTOMERS"', ["CUSTOMERS"]) + expect(row.match).toBe("exact") + expect(row.replaceWith).toBe("{{ source('jaffle', 'customers') }}") + expect(row.candidates).toHaveLength(1) + }) + + test("none: keeps the table as written, no candidates", () => { + const [row] = map("select * from legacy.orders_v1", ["orders_v1"]) + expect(row).toEqual({ raw: "legacy.orders_v1", replaceWith: "legacy.orders_v1", match: "none", candidates: [] }) + }) + + test("a seed is matched with ref()", () => { + const [row] = map("select * from analytics.seeds.countries", ["countries"]) + expect(row.replaceWith).toBe("{{ ref('countries') }}") + expect(row.candidates[0].kind).toBe("seed") + }) + + test("tables that share a last part in different schemas stay separate", () => { + const rows = map("select * from raw.jaffle.customers a join analytics.marts.customers b on 1=1", ["customers"]) + expect(rows.map((r) => [r.raw, r.replaceWith, r.match])).toEqual([ + ["raw.jaffle.customers", "{{ source('jaffle', 'customers') }}", "case_insensitive"], + ["analytics.marts.customers", "{{ ref('customers') }}", "case_insensitive"], + ]) + }) + + test("BigQuery: a backtick path in one piece is split into its parts", () => { + const bq = manifestRelations({ + metadata: { adapter_type: "bigquery", project_name: "shop" }, + nodes: {}, + sources: { + "source.shop.raw.orders": { + resource_type: "source", + source_name: "raw", + name: "orders", + relation_name: "`proj`.`ds`.`orders`", + }, + }, + }) + const [row] = mapRelations(["proj.ds.orders"], "select * from `proj.ds.orders`", bq, "bigquery") + expect(row.match).toBe("exact") + expect(row.replaceWith).toBe("{{ source('raw', 'orders') }}") + }) + + test("a table the text scan does not find is matched by its bare name", () => { + const [row] = map("select 1", ["stg_orders"]) + expect(row.match).toBe("partial") + expect(row.replaceWith).toBe("{{ ref('stg_orders') }}") + }) + + test("a name altimate-core did not report is not a table", () => { + expect(map("with customers as (select 1) select * from customers", [])).toEqual([]) + }) +}) + +const ctx: Tool.Context = { + sessionID: SessionID.make("ses_map_relations"), + messageID: MessageID.make("msg_map_relations"), + agent: "analyst", + abort: new AbortController().signal, + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, + extra: { forceReadOnly: true }, +} + +describe("dbt_map_relations tool", () => { + let dirs: string[] = [] + afterEach(() => { + for (const d of dirs) rmSync(d, { recursive: true, force: true }) + dirs = [] + }) + function manifestPath(manifest: unknown = MANIFEST): string { + const dir = mkdtempSync(join(tmpdir(), "map-relations-")) + dirs.push(dir) + const p = join(dir, "manifest.json") + writeFileSync(p, JSON.stringify(manifest)) + return p + } + async function run(args: Record) { + const tool = await initTool(DbtMapRelationsTool) + return tool.execute(args, ctx) + } + + test("maps every table, takes the dialect from the manifest, returns rows in metadata", async () => { + const result = await run({ + sql: 'select * from raw.jaffle.customers c join "ANALYTICS"."STAGING"."STG_ORDERS" o on 1=1 join legacy.thing t on 1=1 join payments p on 1=1', + manifest_path: manifestPath(), + }) + expect(result.metadata.success).toBe(true) + expect(result.metadata.dialect).toBe("snowflake") + const rows = result.metadata.mappings as { raw: string; replaceWith: string; match: string }[] + expect(rows.map((r) => [r.raw, r.match])).toEqual([ + ["raw.jaffle.customers", "case_insensitive"], + ['"ANALYTICS"."STAGING"."STG_ORDERS"', "exact"], + ["legacy.thing", "none"], + ["payments", "partial"], + ]) + expect(result.output.split("\n")[0]).toBe("none: legacy.thing -> no model or source") + expect(result.title).toContain("1 unmatched") + }) + + test("a missing manifest is an error that names the path", async () => { + const result = await run({ sql: "select * from t", manifest_path: "/nonexistent/manifest.json" }) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("/nonexistent/manifest.json") + }) + + test("SQL that does not parse is an error", async () => { + const result = await run({ sql: "select * from {{ ref('x') }}", manifest_path: manifestPath() }) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("Could not read the tables") + }) + + test("SQL without tables returns no rows", async () => { + const result = await run({ sql: "select 1", manifest_path: manifestPath() }) + expect(result.metadata.success).toBe(true) + expect(result.metadata.mappings).toEqual([]) + expect(result.output).toBe("No tables found in the SQL.") + }) +}) From 8a5c42a7ca840abad99090190bbf7f1e9cf7cce4 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 17:12:22 +0530 Subject: [PATCH 4/8] fix: dbt_map_relations says so when it is run outside a dbt project Co-Authored-By: Claude Sonnet 5.5 --- .../src/altimate/tools/dbt-map-relations.ts | 10 +++--- .../test/altimate/dbt-map-relations.test.ts | 33 ++++++++++++++++++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-map-relations.ts b/packages/opencode/src/altimate/tools/dbt-map-relations.ts index 5036de5852..241416a00f 100644 --- a/packages/opencode/src/altimate/tools/dbt-map-relations.ts +++ b/packages/opencode/src/altimate/tools/dbt-map-relations.ts @@ -159,7 +159,7 @@ function readChain(sql: string, start: number): { raw: string; end: number } | n } if (i >= n) return null i++ - } else if (/[A-Za-z0-9_$\-]/.test(ch) && (i > start || /[A-Za-z_$]/.test(ch))) { + } else if (/[A-Za-z0-9_$-]/.test(ch) && (i > start || /[A-Za-z_$]/.test(ch))) { while (i < n && /[A-Za-z0-9_$]/.test(sql[i])) i++ // BigQuery project ids may contain hyphens (`my-proj.ds.tbl` unquoted). while (sql[i] === "-" && /[A-Za-z0-9_]/.test(sql[i + 1] ?? "")) { @@ -414,6 +414,7 @@ export const DbtMapRelationsTool = Tool.define("dbt_map_relations", { }) try { const manifestPath = args.manifest_path ?? (await defaultManifestPath()) + if (!manifestPath) return fail("No dbt project found here. Run from a dbt project or pass manifest_path.") const manifest = loadRawManifest(manifestPath) if (!manifest) return fail(`No manifest at ${manifestPath}. Run \`dbt parse\` or pass manifest_path.`) const dialect = args.dialect || detectDialect(manifest) @@ -437,13 +438,14 @@ export const DbtMapRelationsTool = Tool.define("dbt_map_relations", { }, }) -async function defaultManifestPath(): Promise { +async function defaultManifestPath(): Promise { const { findProjectRoot } = await import("../../../../dbt-tools/src/config") const found = findProjectRoot(projectRoot()) const root = found ?? (await (await import("../native/connections/register")).getDbtProjectRoot()) - let real = root ?? "" + if (!root) return null + let real = root try { - real = root ? fs.realpathSync(root) : "" + real = fs.realpathSync(root) } catch {} return path.join(real, "target", "manifest.json") } diff --git a/packages/opencode/test/altimate/dbt-map-relations.test.ts b/packages/opencode/test/altimate/dbt-map-relations.test.ts index f568090e77..1f55b1de43 100644 --- a/packages/opencode/test/altimate/dbt-map-relations.test.ts +++ b/packages/opencode/test/altimate/dbt-map-relations.test.ts @@ -6,7 +6,7 @@ * that share it, so the matching rules read the qualifiers back from the SQL text. */ import { describe, test, expect, afterEach } from "bun:test" -import { mkdtempSync, rmSync, writeFileSync } from "fs" +import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "fs" import { tmpdir } from "os" import { join } from "path" import { Effect } from "effect" @@ -324,6 +324,37 @@ describe("dbt_map_relations tool", () => { expect(result.output).toContain("/nonexistent/manifest.json") }) + test("without manifest_path the dbt project around the working directory supplies target/manifest.json", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "map-relations-project-"))) + dirs.push(dir) + writeFileSync(join(dir, "dbt_project.yml"), "name: jaffle\n") + mkdirSync(join(dir, "target")) + writeFileSync(join(dir, "target", "manifest.json"), JSON.stringify(MANIFEST)) + const cwd = process.cwd() + process.chdir(dir) + try { + const result = await run({ sql: "select * from legacy.thing" }) + expect(result.metadata.success).toBe(true) + expect(result.metadata.manifest_path).toBe(join(dir, "target", "manifest.json")) + } finally { + process.chdir(cwd) + } + }) + + test("outside a dbt project, without manifest_path, the error says so", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "map-relations-bare-"))) + dirs.push(dir) + const cwd = process.cwd() + process.chdir(dir) + try { + const result = await run({ sql: "select * from t" }) + expect(result.metadata.success).toBe(false) + expect(result.output).toBe("No dbt project found here. Run from a dbt project or pass manifest_path.") + } finally { + process.chdir(cwd) + } + }) + test("SQL that does not parse is an error", async () => { const result = await run({ sql: "select * from {{ ref('x') }}", manifest_path: manifestPath() }) expect(result.metadata.success).toBe(false) From 0f33f9a68b7a4129c586e604948b0aa8a7af3ca6 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 17:39:37 +0530 Subject: [PATCH 5/8] fix: dbt_map_relations reads the SQL with a tokenizer, handles CTEs and versioned models The table scan now walks tokens instead of matching keywords in text: quoted names, strings and dollar quotes are skipped by dialect, dots may have whitespace or comments around them, tables after a subquery or table function in a FROM list are read, a CTE's name is not a table, and a physical table a CTE shares its name with keeps its qualifiers. Versioned models are referenced with v=. Co-Authored-By: Claude Sonnet 5.5 --- .../src/altimate/tools/dbt-map-relations.ts | 329 ++++++++++++------ .../test/altimate/dbt-map-relations.test.ts | 154 +++++++- 2 files changed, 364 insertions(+), 119 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-map-relations.ts b/packages/opencode/src/altimate/tools/dbt-map-relations.ts index 241416a00f..d297bd1b6e 100644 --- a/packages/opencode/src/altimate/tools/dbt-map-relations.ts +++ b/packages/opencode/src/altimate/tools/dbt-map-relations.ts @@ -91,9 +91,21 @@ export function splitQualified(text: string, splitBacktickDots = false): string[ } /** A table reference found in the SQL text: the text as written and its unquoted parts. */ -type Reference = { raw: string; parts: string[] } +type Reference = { raw: string; parts: string[]; start?: number } + +/** What the scan of a query found: its table references in text order, and the names its CTEs define. */ +export type Scan = { + refs: Reference[] + /** By lower-cased name: whether the WITH is recursive, and the text offsets of each body that defines the name. */ + ctes: Map +} + +// Dialects whose double quotes (and, for some, backslashes) belong to string literals. +const DOUBLE_QUOTE_STRING = new Set(["bigquery", "databricks", "spark", "mysql", "hive"]) +const BACKSLASH_ESCAPE = new Set(["bigquery", "databricks", "spark", "mysql", "hive", "clickhouse", "snowflake"]) +const DOLLAR_QUOTE = new Set(["snowflake", "postgres", "redshift", "duckdb"]) +const BRACKET_QUOTE = new Set(["tsql", "sqlserver", "fabric", "synapse"]) -const IDENT_START = /[A-Za-z_$"`[]/ const RESERVED = new Set([ "select", "from", @@ -133,133 +145,220 @@ const RESERVED = new Set([ "pivot", "unpivot", "tablesample", - "inner", "straight_join", "asof", + "with", + "and", + "or", + "not", + "is", + "in", ]) +// Clause words that start a list of tables. `from` inside these calls reads a column or a string instead. +const TABLE_KEYWORDS = new Set(["from", "join", "into", "update", "using"]) +const FROM_INSIDE_CALL = new Set(["extract", "trim", "substring", "overlay", "position"]) -/** One identifier chain (`a.b.c`, each part quoted or not) starting at `i`; null when there is none. */ -function readChain(sql: string, start: number): { raw: string; end: number } | null { - let i = start +type Token = { kind: "word" | "quoted" | "punct" | "other"; start: number; end: number; text: string } + +/** The tokens of the SQL with comments and string literals dropped; a quoted name stays one token. */ +function tokenize(sql: string, dialect: string): Token[] { + const out: Token[] = [] const n = sql.length - while (i < n) { - const ch = sql[i] - if (ch === '"' || ch === "`" || ch === "[") { - const close = ch === "[" ? "]" : ch - i++ - while (i < n) { - if (sql[i] === close) { - if (sql[i + 1] === close && close !== "]") { - i += 2 - continue - } - break - } - i++ - } - if (i >= n) return null - i++ - } else if (/[A-Za-z0-9_$-]/.test(ch) && (i > start || /[A-Za-z_$]/.test(ch))) { - while (i < n && /[A-Za-z0-9_$]/.test(sql[i])) i++ - // BigQuery project ids may contain hyphens (`my-proj.ds.tbl` unquoted). - while (sql[i] === "-" && /[A-Za-z0-9_]/.test(sql[i + 1] ?? "")) { - i++ - while (i < n && /[A-Za-z0-9_$]/.test(sql[i])) i++ - } - } else return null - if (sql[i] === ".") { - i++ - continue + let i = 0 + const doubleIsString = DOUBLE_QUOTE_STRING.has(dialect) + const backslash = BACKSLASH_ESCAPE.has(dialect) + const push = (kind: Token["kind"], start: number, end: number) => + out.push({ kind, start, end, text: sql.slice(start, end) }) + const skipString = (quote: string, from: number) => { + let j = from + while (j < n) { + if (backslash && sql[j] === "\\") j += 2 + else if (sql[j] === quote) { + if (sql[j + 1] === quote) j += 2 + else return j + 1 + } else j++ } - break + return n } - return i > start ? { raw: sql.slice(start, i), end: i } : null -} - -/** Replace comments and string literals with spaces so the scan never reads inside them. */ -function blank(sql: string): string { - let out = "" - let i = 0 - while (i < sql.length) { + const lastWord = () => { + const last = out[out.length - 1] + return last?.kind === "word" ? last.text.toLowerCase() : "" + } + while (i < n) { + const ch = sql[i] const two = sql.slice(i, i + 2) - if (two === "--") { - while (i < sql.length && sql[i] !== "\n") { - out += " " - i++ - } + if (/\s/.test(ch)) i++ + else if (two === "--") { + while (i < n && sql[i] !== "\n") i++ } else if (two === "/*") { const end = sql.indexOf("*/", i + 2) - const stop = end < 0 ? sql.length : end + 2 - out += sql.slice(i, stop).replace(/[^\n]/g, " ") - i = stop - } else if (sql[i] === "'") { + i = end < 0 ? n : end + 2 + } else if (ch === "'" || (ch === '"' && doubleIsString)) { + const triple = dialect === "bigquery" && sql.startsWith(ch.repeat(3), i) + if (triple) { + const end = sql.indexOf(ch.repeat(3), i + 3) + i = end < 0 ? n : end + 3 + } else i = skipString(ch, i + 1) + } else if ( + ch === "$" && + DOLLAR_QUOTE.has(dialect) && + /^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/.test(sql.slice(i, i + 40)) + ) { + const tag = /^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/.exec(sql.slice(i, i + 40))![0] + const end = sql.indexOf(tag, i + tag.length) + i = end < 0 ? n : end + tag.length + } else if (ch === '"' || ch === "`" || (ch === "[" && BRACKET_QUOTE.has(dialect))) { + const close = ch === "[" ? "]" : ch let j = i + 1 - while (j < sql.length) { - if (sql[j] === "\\") j += 2 - else if (sql[j] === "'") { - if (sql[j + 1] === "'") j += 2 + while (j < n) { + if (sql[j] === close) { + if (sql[j + 1] === close && close !== "]") j += 2 else break } else j++ } - const stop = Math.min(j + 1, sql.length) - out += sql.slice(i, stop).replace(/[^\n]/g, " ") - i = stop - } else if (sql[i] === '"' || sql[i] === "`") { - // Keep quoted identifiers intact; skip over them so a quote inside one is not misread. - const close = sql[i] + const end = Math.min(j + 1, n) + push("quoted", i, end) + i = end + } else if (/[A-Za-z_]/.test(ch)) { + let j = i + 1 + while (j < n && /[A-Za-z0-9_$]/.test(sql[j])) j++ + // BigQuery project ids may contain hyphens (`my-proj.ds.tbl` unquoted), but only where a table is named. + if (dialect === "bigquery" && (TABLE_KEYWORDS.has(lastWord()) || out[out.length - 1]?.text === ",")) { + while (sql[j] === "-" && /[A-Za-z0-9_]/.test(sql[j + 1] ?? "")) { + j++ + while (j < n && /[A-Za-z0-9_$]/.test(sql[j])) j++ + } + } + push("word", i, j) + i = j + } else if (/[0-9]/.test(ch)) { let j = i + 1 - while (j < sql.length && sql[j] !== close) j++ - out += sql.slice(i, Math.min(j + 1, sql.length)) - i = Math.min(j + 1, sql.length) + while (j < n && /[A-Za-z0-9_]/.test(sql[j])) j++ + push("other", i, j) + i = j } else { - out += sql[i] + push(/[.,()]/.test(ch) ? "punct" : "other", i, i + 1) i++ } } return out } -function skipSpace(sql: string, i: number): number { - while (i < sql.length && /\s/.test(sql[i])) i++ - return i +/** The name a quoted token or word holds: quotes removed, doubled quotes undone. */ +function nameOf(token: Token, dialect: string): string[] { + if (token.kind !== "quoted") return [token.text] + const open = token.text[0] + const close = open === "[" ? "]" : open + const body = token.text.slice(1, token.text.endsWith(close) && token.text.length > 1 ? -1 : undefined) + const name = close === "]" ? body : body.split(close + close).join(close) + // BigQuery reads `proj.ds.tbl` in one pair of backticks as a path. + return open === "`" && dialect === "bigquery" ? name.split(".") : [name] } +/** Index of the token after the `)` that closes the `(` at `at`. */ +function afterParens(tokens: Token[], at: number): number { + let depth = 0 + for (let i = at; i < tokens.length; i++) { + if (tokens[i].text === "(" && tokens[i].kind === "punct") depth++ + else if (tokens[i].text === ")" && tokens[i].kind === "punct" && --depth === 0) return i + 1 + } + return tokens.length +} + +const isPunct = (token: Token | undefined, text: string) => token?.kind === "punct" && token.text === text + /** - * The table references of FROM, JOIN, INTO and UPDATE clauses, in text order, written as in the SQL. - * altimate-core's table list keeps only the last name part, so the schema and database a person wrote - * are read back from the text here; the list decides which of these are real tables. + * The table references of FROM, JOIN, INTO, UPDATE and USING clauses, in text order, written as in the SQL, + * and the names the query's CTEs define. altimate-core's table list keeps only the last name part and leaves + * out any table named like a CTE, so the schema and database a person wrote are read back from the text here. */ -export function scanTableReferences(sql: string, splitBacktickDots: boolean): Reference[] { - const text = blank(sql) - const found: Reference[] = [] - const re = /\b(from|join|into|update|using|table)\b/gi - let m: RegExpExecArray | null - while ((m = re.exec(text))) { - let i = skipSpace(text, m.index + m[0].length) +export function scanTableReferences(sql: string, dialect: string): Scan { + const tokens = tokenize(sql, dialect) + const refs: Reference[] = [] + const ctes: Scan["ctes"] = new Map() + const calls: string[] = [] + + // `name [(columns)] as [not] [materialized] (`, then `,` for the next CTE. + const readCtes = (start: number) => { + let i = start + const recursive = tokens[i]?.text.toLowerCase() === "recursive" + if (recursive) i++ + while (tokens[i] && (tokens[i].kind === "word" || tokens[i].kind === "quoted")) { + const name = nameOf(tokens[i], dialect)[0].toLowerCase() + let j = i + 1 + if (isPunct(tokens[j], "(")) j = afterParens(tokens, j) + if (tokens[j]?.text.toLowerCase() !== "as") return + j++ + if (tokens[j]?.text.toLowerCase() === "not") j++ + if (tokens[j]?.text.toLowerCase() === "materialized") j++ + if (!isPunct(tokens[j], "(")) return + const open = tokens[j].start + j = afterParens(tokens, j) + const known = ctes.get(name) ?? { recursive, bodies: [] } + known.bodies.push([open, tokens[j - 1].end]) + ctes.set(name, known) + if (!isPunct(tokens[j], ",")) return + i = j + 1 + } + } + + const readList = (start: number, strict: boolean) => { + let i = start for (;;) { - if (!IDENT_START.test(text[i] ?? "")) break - const chain = readChain(text, i) - if (!chain) break - i = skipSpace(text, chain.end) - // A function call (`flatten(...)`, `unnest(...)`) is not a table. - if (text[i] === "(") break - if (!RESERVED.has(chain.raw.toLowerCase())) { - found.push({ raw: chain.raw, parts: splitQualified(chain.raw, splitBacktickDots) }) - } else break - // An alias, with or without AS, then a comma continues a FROM list. - const asM = /^as\s+/i.exec(text.slice(i)) - if (asM) i = skipSpace(text, i + asM[0].length) - const alias = readChain(text, i) - if (alias && !alias.raw.includes(".") && !RESERVED.has(alias.raw.toLowerCase())) i = skipSpace(text, alias.end) - else if (asM && alias) i = skipSpace(text, alias.end) - if (text[i] === ",") { - i = skipSpace(text, i + 1) - continue + if (isPunct(tokens[i], "(")) i = afterParens(tokens, i) + else { + const first = tokens[i] + if (!first || (first.kind !== "word" && first.kind !== "quoted")) return + if (first.kind === "word" && RESERVED.has(first.text.toLowerCase())) return + const parts: string[] = [...nameOf(first, dialect)] + let at = i + while (isPunct(tokens[at + 1], ".") && ["word", "quoted"].includes(tokens[at + 2]?.kind ?? "")) { + at += 2 + parts.push(...nameOf(tokens[at], dialect)) + } + const last = tokens[at] + i = at + 1 + if (isPunct(tokens[i], "(") && strict) i = afterParens(tokens, i) + else refs.push({ raw: sql.slice(first.start, last.end), parts, start: first.start }) + } + // An alias with or without AS, then its column list, then a comma continues a FROM list. + if (tokens[i]?.text.toLowerCase() === "as") i++ + const alias = tokens[i] + if (alias && (alias.kind === "quoted" || (alias.kind === "word" && !RESERVED.has(alias.text.toLowerCase())))) { + i++ + if (isPunct(tokens[i], "(")) i = afterParens(tokens, i) + } + if (isPunct(tokens[i], ",")) i++ + else return + } + } + + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i] + if (isPunct(token, "(")) { + const before = tokens[i - 1] + calls.push(before?.kind === "word" ? before.text.toLowerCase() : "") + } else if (isPunct(token, ")")) calls.pop() + else if (token.kind === "word") { + const word = token.text.toLowerCase() + if (word === "with") readCtes(i + 1) + else if (TABLE_KEYWORDS.has(word)) { + if ( + word === "from" && + (FROM_INSIDE_CALL.has(calls[calls.length - 1] ?? "") || tokens[i - 1]?.text.toLowerCase() === "distinct") + ) + continue + // A name followed by "(" after FROM, JOIN or USING is a table function; after INTO or UPDATE it is a table. + readList( + word === "into" && tokens[i + 1]?.text.toLowerCase() === "table" ? i + 2 : i + 1, + word === "from" || word === "join" || word === "using", + ) } - break } } - return found + // An outer list is read before the subquery inside it: put them back in text order. + refs.sort((a, b) => a.start! - b.start!) + return { refs, ctes } } /** The model, snapshot, seed and source nodes of a manifest, each with its relation's name parts. */ @@ -288,7 +387,12 @@ export function manifestRelations(manifest: any): Relation[] { typeof node.package_name === "string" && rootProject && node.package_name !== rootProject ? node.package_name : undefined - const call = pkg ? `{{ ref('${pkg}', '${node.name}') }}` : `{{ ref('${node.name}') }}` + // Every version of a model shares its name; a bare ref() resolves to the latest one, so a version is named. + const version = + node.version === null || node.version === undefined || node.version === "" + ? "" + : `, v=${JSON.stringify(node.version)}` + const call = pkg ? `{{ ref('${pkg}', '${node.name}'${version}) }}` : `{{ ref('${node.name}'${version}) }}` out.push({ kind, name: node.name, uniqueId, relationName, parts: splitQualified(relationName, true), call }) } const sources: Record = manifest?.sources ?? {} @@ -319,14 +423,13 @@ export function matchParts(parts: string[], relation: string[]): Exclude t.toLowerCase())) const entries: Reference[] = [] const seen = new Set() const add = (r: Reference) => { - const key = r.parts.join("\u0000") + const key = r.raw.replace(/\s+/g, " ") if (seen.has(key)) return seen.add(key) entries.push(r) @@ -335,14 +438,18 @@ export function mapRelations(tables: string[], sql: string, relations: Relation[ for (const r of refs) { const last = r.parts[r.parts.length - 1] const whole = r.parts.join(".") - if (wanted.has(last) || wanted.has(whole)) { + const cte = ctes.get(last.toLowerCase()) + // A CTE's own name inside its own body is the real table unless the WITH is recursive. + const inOwnBody = !!cte && !cte.recursive && cte.bodies.some(([from, to]) => r.start! >= from && r.start! < to) + if (r.parts.length === 1 && cte && !inOwnBody) continue + if (wanted.has(last.toLowerCase()) || wanted.has(whole.toLowerCase())) { add(r) - covered.add(wanted.has(last) ? last : whole) - } + covered.add((wanted.has(last.toLowerCase()) ? last : whole).toLowerCase()) + } else if (cte) add(r) } for (const t of tables) { - if (covered.has(t)) continue - add({ raw: t, parts: splitQualified(t, splitDots) }) + if (covered.has(t.toLowerCase()) || ctes.has(t.toLowerCase())) continue + add({ raw: t, parts: splitQualified(t, dialect === "bigquery") }) } return entries.map((entry) => { const hits: RelationCandidate[] = [] @@ -417,7 +524,7 @@ export const DbtMapRelationsTool = Tool.define("dbt_map_relations", { if (!manifestPath) return fail("No dbt project found here. Run from a dbt project or pass manifest_path.") const manifest = loadRawManifest(manifestPath) if (!manifest) return fail(`No manifest at ${manifestPath}. Run \`dbt parse\` or pass manifest_path.`) - const dialect = args.dialect || detectDialect(manifest) + const dialect = (args.dialect || detectDialect(manifest)).toLowerCase() const result = await Dispatcher.call("altimate_core.metadata", { sql: args.sql, dialect }) const data = (result.data ?? {}) as Record const error = result.error ?? data.error diff --git a/packages/opencode/test/altimate/dbt-map-relations.test.ts b/packages/opencode/test/altimate/dbt-map-relations.test.ts index 1f55b1de43..83331de958 100644 --- a/packages/opencode/test/altimate/dbt-map-relations.test.ts +++ b/packages/opencode/test/altimate/dbt-map-relations.test.ts @@ -84,23 +84,79 @@ describe("splitQualified", () => { }) describe("scanTableReferences", () => { + const raws = (sql: string, dialect = "snowflake") => scanTableReferences(sql, dialect).refs.map((r) => r.raw) + test("reads qualified names from FROM, JOIN and comma lists, written as in the SQL", () => { - const refs = scanTableReferences( + const { refs } = scanTableReferences( 'select * from raw.jaffle.customers c, "RAW"."JAFFLE"."orders" as o left join other.payments p on 1=1', - false, + "snowflake", ) expect(refs.map((r) => r.raw)).toEqual(["raw.jaffle.customers", '"RAW"."JAFFLE"."orders"', "other.payments"]) expect(refs[1].parts).toEqual(["RAW", "JAFFLE", "orders"]) }) test("ignores table functions, subqueries, comments and strings", () => { - const refs = scanTableReferences( - "select 'from fake.t' as s from table(flatten(x)) f join (select 1) q on 1=1 /* from c.d */ -- from e.f\n join real.t t on 1=1", - false, - ) - expect(refs.map((r) => r.raw)).toEqual(["real.t"]) + expect( + raws( + "select 'from fake.t' as s from table(flatten(x)) f join (select 1) q on 1=1 /* from c.d */ -- from e.f\n join real.t t on 1=1", + ), + ).toEqual(["real.t"]) }) test("reads hyphenated BigQuery project ids", () => { - expect(scanTableReferences("select * from my-proj.ds.tbl", true)[0].parts).toEqual(["my-proj", "ds", "tbl"]) + expect(scanTableReferences("select * from my-proj.ds.tbl", "bigquery").refs[0].parts).toEqual([ + "my-proj", + "ds", + "tbl", + ]) + }) + test("whitespace or a comment around a dot does not drop the qualifier", () => { + expect(raws("select * from db . right . orders")).toEqual(["db . right . orders"]) + expect(scanTableReferences("select * from other/*gap*/.marts.customers", "snowflake").refs[0].parts).toEqual([ + "other", + "marts", + "customers", + ]) + }) + test("a table after a subquery or table function in a comma list is still read", () => { + expect(raws("select * from (select 1) x, db.right.orders")).toEqual(["db.right.orders"]) + expect(raws("select * from table(generator(rowcount => 1)) x, db.right.orders")).toEqual(["db.right.orders"]) + expect(raws("select * from (select * from db.wrong.orders) x, db.right.orders")).toEqual([ + "db.wrong.orders", + "db.right.orders", + ]) + }) + test("brackets quote names for SQL Server dialects only; ClickHouse double quotes are identifiers", () => { + expect(raws("select * from [db].[s].[t]", "tsql")).toEqual(["[db].[s].[t]"]) + expect(raws('select * from "db"."orders"', "clickhouse")).toEqual(['"db"."orders"']) + }) + test("a hyphenated BigQuery project id is read after a comma and `insert into table` names its table", () => { + expect(raws("select * from a, my-proj.ds.t", "bigquery")).toEqual(["a", "my-proj.ds.t"]) + expect(raws("insert into table db.t select 1", "databricks")).toEqual(["db.t"]) + }) + test("clause words inside quoted names, strings and dollar quotes are not clauses", () => { + expect(raws('select "from orders", "join orders" as "using orders" from db.right.orders')).toEqual([ + "db.right.orders", + ]) + expect(raws("select $$from orders $$ as s from db.right.orders")).toEqual(["db.right.orders"]) + expect(raws("select $q1$from orders $q1$ as s from db.right.orders", "postgres")).toEqual(["db.right.orders"]) + expect(raws('select "from orders", r"from orders " from db.right.orders', "bigquery")).toEqual(["db.right.orders"]) + expect(raws('select "from orders" from db.right.orders', "databricks")).toEqual(["db.right.orders"]) + }) + test("EXTRACT, TRIM and IS DISTINCT FROM are not table clauses", () => { + expect(raws("select extract(year from d), trim(both 'x' from s) from t where a is distinct from orders")).toEqual([ + "t", + ]) + }) + test("a backslash ends a string in dialects where it does not escape", () => { + expect(raws("select 'C:\\' as p from db.right.orders", "postgres")).toEqual(["db.right.orders"]) + expect(raws("select 'it\\'s' as p from db.right.orders", "snowflake")).toEqual(["db.right.orders"]) + }) + test("CTE names are collected, and a table named like a CTE keeps its qualifiers", () => { + const { refs, ctes } = scanTableReferences( + "with orders as (select * from db.raw.orders), b(x) as materialized (select 1) select * from orders", + "snowflake", + ) + expect([...ctes.keys()]).toEqual(["orders", "b"]) + expect(refs.map((r) => r.raw)).toEqual(["db.raw.orders", "orders"]) }) }) @@ -266,6 +322,68 @@ describe("mapRelations", () => { expect(row.replaceWith).toBe("{{ ref('stg_orders') }}") }) + test("a CTE named like a table is not mapped, and the physical table it reads keeps its mapping", () => { + const sql = "with customers as (select * from raw.jaffle.customers) select * from customers" + expect(map(sql, []).map((r) => [r.raw, r.match])).toEqual([["raw.jaffle.customers", "case_insensitive"]]) + expect(map(sql, ["customers"]).map((r) => [r.raw, r.match])).toEqual([["raw.jaffle.customers", "case_insensitive"]]) + }) + + test("a table named like its own CTE, read inside that CTE, is the real table; a recursive CTE's own name is not", () => { + expect(map("with orders as (select * from orders) select * from orders", []).map((r) => r.raw)).toEqual(["orders"]) + expect( + map("with recursive customers as (select 1 union all select * from customers) select * from customers", []), + ).toEqual([]) + }) + + test("differently written forms of one table stay separate rows; an identical repeat does not", () => { + const sql = + 'select * from raw.jaffle.payments a join "RAW"."JAFFLE"."PAYMENTS" b on 1=1 join raw.jaffle.payments c on 1=1' + expect(map(sql, ["PAYMENTS"]).map((r) => r.raw)).toEqual(["raw.jaffle.payments", '"RAW"."JAFFLE"."PAYMENTS"']) + }) + + test("a versioned model is referenced by its version", () => { + const versioned = manifestRelations({ + metadata: { project_name: "shop" }, + nodes: { + "model.shop.orders.v1": { + resource_type: "model", + name: "orders", + package_name: "shop", + version: 1, + relation_name: '"A"."B"."ORDERS_V1"', + config: {}, + }, + "model.shop.orders.v2": { + resource_type: "model", + name: "orders", + package_name: "shop", + version: "2", + relation_name: '"A"."B"."ORDERS"', + config: {}, + }, + "model.pkg.dim.v3": { + resource_type: "model", + name: "dim", + package_name: "pkg", + version: 3, + relation_name: '"A"."B"."DIM"', + config: {}, + }, + }, + }) + const rows = mapRelations( + ["ORDERS_V1", "ORDERS", "DIM"], + 'select * from "A"."B"."ORDERS_V1", "A"."B"."ORDERS", "A"."B"."DIM"', + versioned, + "snowflake", + ) + expect(rows.map((r) => r.replaceWith)).toEqual([ + "{{ ref('orders', v=1) }}", + "{{ ref('orders', v=\"2\") }}", + "{{ ref('pkg', 'dim', v=3) }}", + ]) + }) + test("a name altimate-core did not report is not a table", () => { expect(map("with customers as (select 1) select * from customers", [])).toEqual([]) }) @@ -331,6 +449,8 @@ describe("dbt_map_relations tool", () => { mkdirSync(join(dir, "target")) writeFileSync(join(dir, "target", "manifest.json"), JSON.stringify(MANIFEST)) const cwd = process.cwd() + const home = process.env.HOME + process.env.HOME = dir process.chdir(dir) try { const result = await run({ sql: "select * from legacy.thing" }) @@ -338,6 +458,8 @@ describe("dbt_map_relations tool", () => { expect(result.metadata.manifest_path).toBe(join(dir, "target", "manifest.json")) } finally { process.chdir(cwd) + if (home === undefined) delete process.env.HOME + else process.env.HOME = home } }) @@ -345,6 +467,8 @@ describe("dbt_map_relations tool", () => { const dir = realpathSync(mkdtempSync(join(tmpdir(), "map-relations-bare-"))) dirs.push(dir) const cwd = process.cwd() + const home = process.env.HOME + process.env.HOME = dir process.chdir(dir) try { const result = await run({ sql: "select * from t" }) @@ -352,9 +476,23 @@ describe("dbt_map_relations tool", () => { expect(result.output).toBe("No dbt project found here. Run from a dbt project or pass manifest_path.") } finally { process.chdir(cwd) + if (home === undefined) delete process.env.HOME + else process.env.HOME = home } }) + test("with the real parser, a table a CTE shares its name with and a dot spaced out are both mapped", async () => { + const result = await run({ + sql: "with customers as (select * from raw.jaffle.customers) select * from customers c join analytics . staging . stg_orders o on 1=1", + manifest_path: manifestPath(), + }) + const rows = result.metadata.mappings as { raw: string; match: string }[] + expect(rows.map((r) => [r.raw, r.match])).toEqual([ + ["raw.jaffle.customers", "case_insensitive"], + ["analytics . staging . stg_orders", "case_insensitive"], + ]) + }) + test("SQL that does not parse is an error", async () => { const result = await run({ sql: "select * from {{ ref('x') }}", manifest_path: manifestPath() }) expect(result.metadata.success).toBe(false) From acd8faa7a6f08f861e1df1c35cf07557d1a3799c Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 21:33:10 +0530 Subject: [PATCH 6/8] fix: dbt_map_relations tests register the parser handlers and the placeholder test skips the implemented tool Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/test/altimate/dbt-adapter-access.test.ts | 4 +++- packages/opencode/test/altimate/dbt-map-relations.test.ts | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index b2a012ed58..a0b84aa447 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -30,6 +30,8 @@ const MINIMAL_ARGS: Record> = { dbt_cte_profile: { model: "customers" }, } const FEATURE_TOOLS = Object.keys(MINIMAL_ARGS) +/** Already implemented, so it is not a placeholder. */ +const IMPLEMENTED_TOOL = "dbt_map_relations" const ctx: Tool.Context = { sessionID: SessionID.make("ses_dbt_adapter_access"), @@ -82,7 +84,7 @@ describe("feature tools in the allow-list", () => { }) test("each placeholder answers that it isn't implemented, as a failure", async () => { - for (const id of FEATURE_TOOLS) { + for (const id of FEATURE_TOOLS.filter((id) => id !== IMPLEMENTED_TOOL)) { const tool = await initTool(EXPOSED[id] as any) expect(tool.id).toBe(id) const result = await tool.execute(MINIMAL_ARGS[id], ctx) diff --git a/packages/opencode/test/altimate/dbt-map-relations.test.ts b/packages/opencode/test/altimate/dbt-map-relations.test.ts index 83331de958..de23eed06e 100644 --- a/packages/opencode/test/altimate/dbt-map-relations.test.ts +++ b/packages/opencode/test/altimate/dbt-map-relations.test.ts @@ -5,7 +5,7 @@ * qualified names: it keeps only the last name part (quotes removed, case kept) and merges tables * that share it, so the matching rules read the qualifiers back from the SQL text. */ -import { describe, test, expect, afterEach } from "bun:test" +import { describe, test, expect, afterEach, beforeAll } from "bun:test" import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "fs" import { tmpdir } from "os" import { join } from "path" @@ -13,6 +13,7 @@ import { Effect } from "effect" import type { Tool } from "@/tool/tool" import { SessionID, MessageID } from "@/session/schema" import { Dispatcher } from "../../src/altimate/native" +import { registerAll as registerAltimateCore } from "../../src/altimate/native/altimate-core" import { initTool } from "./tool-fixture" import { DbtMapRelationsTool, @@ -23,6 +24,9 @@ import { splitQualified, } from "../../src/altimate/tools/dbt-map-relations" +// Other test files reset the dispatcher; register the parser handlers this file depends on. +beforeAll(() => registerAltimateCore()) + async function tablesOf(sql: string, dialect: string): Promise { const result = await Dispatcher.call("altimate_core.metadata", { sql, dialect }) expect(result.success).toBe(true) From efc72f6204bbf2aaa97ec5ea691c774b4350b0fc Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Sun, 15 Nov 2026 22:13:58 +0530 Subject: [PATCH 7/8] fix: dbt_map_relations escapes names in generated calls, splits backtick dots only for BigQuery, continues past LATERAL and keeps quoted whitespace Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/src/altimate/api/client.ts | 1 + .../src/altimate/tools/dbt-adapter-access.ts | 1 + .../src/altimate/tools/dbt-map-relations.ts | 28 ++++++++++++---- .../test/altimate/dbt-map-relations.test.ts | 32 +++++++++++++++++++ 4 files changed, 55 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index cb53c0ec7b..76f91303af 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -353,6 +353,7 @@ export namespace AltimateApi { * message naming the credentials file when none are stored or they can't be read. */ export async function apiRequest(method: string, endpoint: string, body?: unknown): Promise { + // The body is whatever JSON the endpoint returns; each endpoint function in other files types its own result. let creds: AltimateCredentials try { creds = await getCredentials() diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index d73dd7e9c5..4b46a7fbf6 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -8,6 +8,7 @@ export type DbtAdapterRefusal = "read_only" | "other_project" | "no_project" | " /** The project's dbt adapter, or why this call may not use it (`reason` is a sentence for the user). */ export type DbtAdapterAccess = + // The adapter is the untyped dbt integration object from the dbt-integration package. | { adapter: any; refusal?: undefined; reason?: undefined } | { adapter?: undefined; refusal: DbtAdapterRefusal; reason: string } diff --git a/packages/opencode/src/altimate/tools/dbt-map-relations.ts b/packages/opencode/src/altimate/tools/dbt-map-relations.ts index d297bd1b6e..137e47edcc 100644 --- a/packages/opencode/src/altimate/tools/dbt-map-relations.ts +++ b/packages/opencode/src/altimate/tools/dbt-map-relations.ts @@ -305,6 +305,8 @@ export function scanTableReferences(sql: string, dialect: string): Scan { const readList = (start: number, strict: boolean) => { let i = start for (;;) { + // LATERAL prefixes a table expression in the list; the list goes on after it. + if (tokens[i]?.kind === "word" && tokens[i].text.toLowerCase() === "lateral") i++ if (isPunct(tokens[i], "(")) i = afterParens(tokens, i) else { const first = tokens[i] @@ -361,9 +363,17 @@ export function scanTableReferences(sql: string, dialect: string): Scan { return { refs, ctes } } +/** A single-quoted Jinja string literal. */ +function jinjaString(value: string): string { + return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'` +} + /** The model, snapshot, seed and source nodes of a manifest, each with its relation's name parts. */ +// The manifest is parsed JSON of a shape dbt owns and versions; its fields are checked where they are read. export function manifestRelations(manifest: any): Relation[] { const rootProject: string | undefined = manifest?.metadata?.project_name + // Only BigQuery names put dots inside backticks; elsewhere a dot in a quoted name is part of the name. + const splitDots = detectDialect(manifest).toLowerCase() === "bigquery" const out: Relation[] = [] const relationOf = (node: any): string | undefined => { if (typeof node?.relation_name === "string" && node.relation_name) return node.relation_name @@ -392,8 +402,10 @@ export function manifestRelations(manifest: any): Relation[] { node.version === null || node.version === undefined || node.version === "" ? "" : `, v=${JSON.stringify(node.version)}` - const call = pkg ? `{{ ref('${pkg}', '${node.name}'${version}) }}` : `{{ ref('${node.name}'${version}) }}` - out.push({ kind, name: node.name, uniqueId, relationName, parts: splitQualified(relationName, true), call }) + const call = pkg + ? `{{ ref(${jinjaString(pkg)}, ${jinjaString(node.name)}${version}) }}` + : `{{ ref(${jinjaString(node.name)}${version}) }}` + out.push({ kind, name: node.name, uniqueId, relationName, parts: splitQualified(relationName, splitDots), call }) } const sources: Record = manifest?.sources ?? {} for (const [uniqueId, node] of Object.entries(sources)) { @@ -404,8 +416,8 @@ export function manifestRelations(manifest: any): Relation[] { name: node.name, uniqueId, relationName, - parts: splitQualified(relationName, true), - call: `{{ source('${node.source_name}', '${node.name}') }}`, + parts: splitQualified(relationName, splitDots), + call: `{{ source(${jinjaString(node.source_name)}, ${jinjaString(node.name)}) }}`, }) } return out @@ -418,7 +430,9 @@ export function matchParts(parts: string[], relation: string[]): Exclude p === tail[i]) const fold = parts.every((p, i) => p.toLowerCase() === tail[i].toLowerCase()) if (parts.length < relation.length) return fold ? "partial" : null - return same ? "exact" : fold ? "case_insensitive" : null + if (same) return "exact" + if (fold) return "case_insensitive" + return null } /** Map each table of `sql` to the manifest's ref() or source() call that builds it. */ @@ -429,7 +443,7 @@ export function mapRelations(tables: string[], sql: string, relations: Relation[ const entries: Reference[] = [] const seen = new Set() const add = (r: Reference) => { - const key = r.raw.replace(/\s+/g, " ") + const key = r.raw if (seen.has(key)) return seen.add(key) entries.push(r) @@ -552,7 +566,7 @@ async function defaultManifestPath(): Promise { if (!root) return null let real = root try { - real = fs.realpathSync(root) + real = await fs.promises.realpath(root) } catch {} return path.join(real, "target", "manifest.json") } diff --git a/packages/opencode/test/altimate/dbt-map-relations.test.ts b/packages/opencode/test/altimate/dbt-map-relations.test.ts index de23eed06e..418b0a9510 100644 --- a/packages/opencode/test/altimate/dbt-map-relations.test.ts +++ b/packages/opencode/test/altimate/dbt-map-relations.test.ts @@ -510,3 +510,35 @@ describe("dbt_map_relations tool", () => { expect(result.output).toBe("No tables found in the SQL.") }) }) + +describe("review regressions", () => { + test("a name with a quote or backslash is escaped in the generated call", () => { + const rel = manifestRelations({ + nodes: { "model.p.o'x": { resource_type: "model", name: "o'x", schema: "s", database: "d", package_name: "p" } }, + sources: { "source.p.a.b": { source_name: "a\\b", name: "b'c", schema: "s", database: "d" } }, + }) + expect(rel.find((r) => r.kind === "model")?.call).toBe("{{ ref('o\\'x') }}") + expect(rel.find((r) => r.kind === "source")?.call).toBe("{{ source('a\\\\b', 'b\\'c') }}") + }) + + test("a dot inside backticks is split for BigQuery manifests only", () => { + const node = { resource_type: "model", name: "orders", relation_name: "`main`.`raw`.`orders.v1`", schema: "raw" } + const dbx = manifestRelations({ metadata: { adapter_type: "databricks" }, nodes: { "model.p.orders": node } }) + expect(dbx[0].parts).toEqual(["main", "raw", "orders.v1"]) + const bq = manifestRelations({ metadata: { adapter_type: "bigquery" }, nodes: { "model.p.orders": node } }) + expect(bq[0].parts).toEqual(["main", "raw", "orders", "v1"]) + }) + + test("the table list goes on after LATERAL", () => { + const { refs } = scanTableReferences( + "select * from db.one.orders a, lateral flatten(input => a.items) f, db.two.orders b", + "snowflake", + ) + expect(refs.map((r) => r.raw)).toEqual(["db.one.orders", "db.two.orders"]) + }) + + test("quoted names that differ only in inner whitespace are different tables", () => { + const rows = mapRelations(['"A B"', '"A B"'], 'select * from "A B", "A B"', [], "snowflake") + expect(rows.map((r) => r.raw)).toEqual(['"A B"', '"A B"']) + }) +}) From 18f72640b9770c22e60a6f7452ca3ee1c9ab05ba Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Sun, 15 Nov 2026 22:20:44 +0530 Subject: [PATCH 8/8] fix: dbt_map_relations splits backtick dots by the dialect it resolved and tolerates a null manifest Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/src/altimate/tools/dbt-map-relations.ts | 6 +++--- packages/opencode/test/altimate/dbt-map-relations.test.ts | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-map-relations.ts b/packages/opencode/src/altimate/tools/dbt-map-relations.ts index 137e47edcc..e9f49e5106 100644 --- a/packages/opencode/src/altimate/tools/dbt-map-relations.ts +++ b/packages/opencode/src/altimate/tools/dbt-map-relations.ts @@ -370,10 +370,10 @@ function jinjaString(value: string): string { /** The model, snapshot, seed and source nodes of a manifest, each with its relation's name parts. */ // The manifest is parsed JSON of a shape dbt owns and versions; its fields are checked where they are read. -export function manifestRelations(manifest: any): Relation[] { +export function manifestRelations(manifest: any, dialect?: string): Relation[] { const rootProject: string | undefined = manifest?.metadata?.project_name // Only BigQuery names put dots inside backticks; elsewhere a dot in a quoted name is part of the name. - const splitDots = detectDialect(manifest).toLowerCase() === "bigquery" + const splitDots = (dialect ?? detectDialect(manifest ?? {})).toLowerCase() === "bigquery" const out: Relation[] = [] const relationOf = (node: any): string | undefined => { if (typeof node?.relation_name === "string" && node.relation_name) return node.relation_name @@ -546,7 +546,7 @@ export const DbtMapRelationsTool = Tool.define("dbt_map_relations", { const tables: string[] = Array.isArray(data.tables) ? data.tables.filter((t: unknown) => typeof t === "string") : [] - const mappings = mapRelations(tables, args.sql, manifestRelations(manifest), dialect) + const mappings = mapRelations(tables, args.sql, manifestRelations(manifest, dialect), dialect) const count = (m: RelationMatch) => mappings.filter((r) => r.match === m).length return { title: `dbt Map Relations: ${mappings.length} tables, ${count("none")} unmatched`, diff --git a/packages/opencode/test/altimate/dbt-map-relations.test.ts b/packages/opencode/test/altimate/dbt-map-relations.test.ts index 418b0a9510..eb32e2fbac 100644 --- a/packages/opencode/test/altimate/dbt-map-relations.test.ts +++ b/packages/opencode/test/altimate/dbt-map-relations.test.ts @@ -529,6 +529,13 @@ describe("review regressions", () => { expect(bq[0].parts).toEqual(["main", "raw", "orders", "v1"]) }) + test("an explicit dialect decides the backtick split, and a null manifest does not throw", () => { + const node = { resource_type: "model", name: "orders", relation_name: "`p.ds.orders`", schema: "ds" } + const rel = manifestRelations({ nodes: { "model.p.orders": node } }, "bigquery") + expect(rel[0].parts).toEqual(["p", "ds", "orders"]) + expect(manifestRelations(null)).toEqual([]) + }) + test("the table list goes on after LATERAL", () => { const { refs } = scanTableReferences( "select * from db.one.orders a, lateral flatten(input => a.items) f, db.two.orders b",