From dd7bcdc547fabc8b6f9f873d90f6b86632cbfaea Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Tue, 15 Sep 2026 23:06:00 -0400 Subject: [PATCH 1/8] Add transform-test support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A transform test pins a transform's behaviour without touching real data: every table it reads is replaced by an input fixture, the transform runs into a temp table, and each expectation checks that output. `transform-test list/get/create/update/delete/run` over `/api/transform-test`, with the schemas the endpoint speaks — inputs as SQL or inline rows, expectations as `empty` or `equals`, and a run report naming what each expectation found. `run` exits non-zero when the test does not pass, so it drops into CI as it stands. Gated at v64, the first version serving the endpoint. Co-Authored-By: Claude Opus 5 --- README.md | 61 +++++ packages/cli/skill-data/core/SKILL.md | 2 +- .../cli/src/commands/transform-test/create.ts | 38 +++ .../cli/src/commands/transform-test/delete.ts | 31 +++ .../cli/src/commands/transform-test/get.ts | 26 ++ .../cli/src/commands/transform-test/index.ts | 14 + .../cli/src/commands/transform-test/list.ts | 31 +++ .../cli/src/commands/transform-test/run.ts | 61 +++++ .../cli/src/commands/transform-test/update.ts | 36 +++ packages/cli/src/main.ts | 1 + .../cli/src/output/views/transform-test.ts | 13 + packages/cli/src/runtime/command-help.test.ts | 24 ++ packages/client/src/client.ts | 2 + packages/client/src/domain/transform-test.ts | 170 +++++++++++++ packages/client/src/index.test.ts | 6 + packages/client/src/index.ts | 11 + .../src/resources/transform-test.test.ts | 239 ++++++++++++++++++ .../client/src/resources/transform-test.ts | 86 +++++++ 18 files changed, 851 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/commands/transform-test/create.ts create mode 100644 packages/cli/src/commands/transform-test/delete.ts create mode 100644 packages/cli/src/commands/transform-test/get.ts create mode 100644 packages/cli/src/commands/transform-test/index.ts create mode 100644 packages/cli/src/commands/transform-test/list.ts create mode 100644 packages/cli/src/commands/transform-test/run.ts create mode 100644 packages/cli/src/commands/transform-test/update.ts create mode 100644 packages/cli/src/output/views/transform-test.ts create mode 100644 packages/client/src/domain/transform-test.ts create mode 100644 packages/client/src/resources/transform-test.test.ts create mode 100644 packages/client/src/resources/transform-test.ts diff --git a/README.md b/README.md index 04aba583..5fabe33c 100644 --- a/README.md +++ b/README.md @@ -379,6 +379,67 @@ mb transform-tag delete 5 --yes | ------- | --------------------------------------------------------------------------------------------------------------------------------- | | `--yes` | Skip the interactive confirmation prompt. In non-TTY contexts the prompt is skipped automatically (kubectl/gh/docker convention). | +## Transform tests + +CRUD and run on `/api/transform-test`. Requires Metabase v64 or newer. A transform test pins a transform's behaviour without touching real data: every table the transform reads is replaced by an `input` fixture, the transform runs into a temp table, and each `expectation` checks that output. The temp tables are dropped when the run ends. + +An input names its `table` and carries either `format: "sql"` with a `sql` query or `format: "rows"` with `columns` (each with a `database_type` the warehouse accepts as a `CAST` target) and `rows`. An expectation is either `type: "empty"` with the `sql` that must return no rows, or `type: "equals"` with the rows the output must hold exactly. + +### `mb transform-test list` + +```sh +mb transform-test list --json +mb transform-test list --transform 1 +``` + +| Flag | Description | +| ------------------ | ------------------------------------ | +| `--transform ` | Only the tests of this transform id. | + +### `mb transform-test get ` + +```sh +mb transform-test get 1 --json +``` + +### `mb transform-test create` + +```sh +mb transform-test create --file transform-test.json +``` + +| Flag | Description | +| --------------- | ----------------------- | +| `--body ` | Inline JSON body. | +| `--file ` | Path to JSON body file. | + +### `mb transform-test update ` + +Only the fields the body carries are patched; `inputs` and `expectations` replace what is stored rather than merging into it. + +```sh +mb transform-test update 1 --body '{"name":"renamed"}' +``` + +### `mb transform-test delete ` + +```sh +mb transform-test delete 1 --yes +``` + +| Flag | Description | +| ------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `--yes` | Skip the interactive confirmation prompt. In non-TTY contexts the prompt is skipped automatically (kubectl/gh/docker convention). | + +### `mb transform-test run ` + +Runs the transform against the fixtures and reports what each expectation found. Exits non-zero when the test does not pass, so it drops straight into CI. + +```sh +mb transform-test run 1 +mb transform-test run 1 --json +``` + ## Databases Read warehouse metadata from `/api/database`. The `db` group exposes the full database list, the per-database record with optional table/field hydration, schema and table inspection, and the two manual-sync triggers. diff --git a/packages/cli/skill-data/core/SKILL.md b/packages/cli/skill-data/core/SKILL.md index 49e0ebff..6d30d1cb 100644 --- a/packages/cli/skill-data/core/SKILL.md +++ b/packages/cli/skill-data/core/SKILL.md @@ -12,7 +12,7 @@ Top-level command groups (run `mb --help` to discover verbs): ``` auth | db | table | field | upload | content-translation | query | card | dashboard | snippet | segment | measure | collection | library -document | timeline | timeline-event | transform | transform-job | transform-tag | alert | subscription | setting +document | timeline | timeline-event | transform | transform-job | transform-tag | transform-test | alert | subscription | setting search | git-sync | setup | eid | uuid | upgrade | skills ``` diff --git a/packages/cli/src/commands/transform-test/create.ts b/packages/cli/src/commands/transform-test/create.ts new file mode 100644 index 00000000..50487782 --- /dev/null +++ b/packages/cli/src/commands/transform-test/create.ts @@ -0,0 +1,38 @@ +import { TransformTest, TransformTestCreateInput } from "@metabase/client/domain/transform-test"; + +import { renderSummary } from "../../output/render"; +import { transformTestView } from "../../output/views/transform-test"; +import { readBody } from "../../runtime/body"; +import { bodyInputFlags } from "../body-flags"; +import { connectionFlags, outputFlags, profileFlag } from "../flags"; +import { defineMetabaseCommand } from "../runtime"; + +export default defineMetabaseCommand({ + meta: { name: "create", description: "Create a transform test from JSON" }, + details: + 'The JSON body needs a `transform_id`, a `name`, the `inputs` standing in for every table the transform reads, and the `expectations` checked against its output. An input names its `table` and carries either `format: "sql"` with a `sql` query or `format: "rows"` with `columns` and `rows`. An expectation is either `type: "empty"` with the `sql` that must return nothing, or `type: "equals"` with the rows the output must hold.', + capabilities: { minVersion: 64 }, + args: { + ...outputFlags, + ...profileFlag, + ...connectionFlags, + ...bodyInputFlags, + }, + inputSchema: TransformTestCreateInput, + outputSchema: TransformTest, + examples: [ + "cat transform-test.json | mb transform-test create", + "mb transform-test create --file transform-test.json", + ], + async run({ args, ctx, getClient }) { + const body = await readBody({ flag: args.body, file: args.file }, TransformTestCreateInput); + const client = await getClient(); + const created = await client.transformTest.create(body); + renderSummary( + created, + transformTestView, + `Created transform test ${created.id} "${created.name}".`, + ctx, + ); + }, +}); diff --git a/packages/cli/src/commands/transform-test/delete.ts b/packages/cli/src/commands/transform-test/delete.ts new file mode 100644 index 00000000..e057deeb --- /dev/null +++ b/packages/cli/src/commands/transform-test/delete.ts @@ -0,0 +1,31 @@ +import { confirmAndDelete, DeleteResult } from "../delete-runtime"; +import { connectionFlags, outputFlags, profileFlag } from "../flags"; +import { parseId } from "../parse-id"; +import { defineMetabaseCommand } from "../runtime"; + +export default defineMetabaseCommand({ + meta: { name: "delete", description: "Delete a transform test by id" }, + capabilities: { minVersion: 64 }, + args: { + ...outputFlags, + ...profileFlag, + ...connectionFlags, + yes: { type: "boolean", description: "Skip confirmation", default: false }, + id: { type: "positional", description: "Transform test id", required: true }, + }, + outputSchema: DeleteResult, + examples: ["mb transform-test delete 1 --yes", "mb transform-test delete 1"], + async run({ args, ctx, getClient }) { + const id = parseId(args.id); + const client = await getClient(); + await confirmAndDelete({ + id, + yes: args.yes, + promptMessage: `Delete transform test ${id}?`, + successMessage: `Deleted transform test ${id}.`, + abortMessage: `Aborted; transform test ${id} was not deleted.`, + deleteResource: () => client.transformTest.delete(id), + ctx, + }); + }, +}); diff --git a/packages/cli/src/commands/transform-test/get.ts b/packages/cli/src/commands/transform-test/get.ts new file mode 100644 index 00000000..66e46c96 --- /dev/null +++ b/packages/cli/src/commands/transform-test/get.ts @@ -0,0 +1,26 @@ +import { TransformTest } from "@metabase/client/domain/transform-test"; + +import { renderItem } from "../../output/render"; +import { transformTestView } from "../../output/views/transform-test"; +import { connectionFlags, outputFlags, profileFlag } from "../flags"; +import { parseId } from "../parse-id"; +import { defineMetabaseCommand } from "../runtime"; + +export default defineMetabaseCommand({ + meta: { name: "get", description: "Get a transform test by id" }, + capabilities: { minVersion: 64 }, + args: { + ...outputFlags, + ...profileFlag, + ...connectionFlags, + id: { type: "positional", description: "Transform test id", required: true }, + }, + outputSchema: TransformTest, + examples: ["mb transform-test get 1", "mb transform-test get 1 --json"], + async run({ args, ctx, getClient }) { + const id = parseId(args.id); + const client = await getClient(); + const transformTest = await client.transformTest.get(id); + renderItem(transformTest, transformTestView, ctx); + }, +}); diff --git a/packages/cli/src/commands/transform-test/index.ts b/packages/cli/src/commands/transform-test/index.ts new file mode 100644 index 00000000..757ae02b --- /dev/null +++ b/packages/cli/src/commands/transform-test/index.ts @@ -0,0 +1,14 @@ +import { defineCommandGroup } from "../group"; + +export default defineCommandGroup({ + name: "transform-test", + description: "Manage transform tests — fixtures and expectations run against temp tables", + subCommands: { + list: () => import("./list").then((mod) => mod.default), + get: () => import("./get").then((mod) => mod.default), + create: () => import("./create").then((mod) => mod.default), + update: () => import("./update").then((mod) => mod.default), + delete: () => import("./delete").then((mod) => mod.default), + run: () => import("./run").then((mod) => mod.default), + }, +}); diff --git a/packages/cli/src/commands/transform-test/list.ts b/packages/cli/src/commands/transform-test/list.ts new file mode 100644 index 00000000..059e3c59 --- /dev/null +++ b/packages/cli/src/commands/transform-test/list.ts @@ -0,0 +1,31 @@ +import { TransformTestCompact } from "@metabase/client/domain/transform-test"; + +import { renderList } from "../../output/render"; +import { listEnvelopeSchema } from "../../output/types"; +import { windowList } from "../../output/window"; +import { transformTestView } from "../../output/views/transform-test"; +import { connectionFlags, listFlags, outputFlags, profileFlag } from "../flags"; +import { parseId } from "../parse-id"; +import { defineMetabaseCommand } from "../runtime"; + +export const TransformTestListEnvelope = listEnvelopeSchema(TransformTestCompact); + +export default defineMetabaseCommand({ + meta: { name: "list", description: "List transform tests" }, + capabilities: { minVersion: 64 }, + args: { + ...outputFlags, + ...listFlags, + ...profileFlag, + ...connectionFlags, + transform: { type: "string", description: "Only the tests of this transform id" }, + }, + outputSchema: TransformTestListEnvelope, + examples: ["mb transform-test list", "mb transform-test list --transform 1 --json"], + async run({ args, ctx, getClient }) { + const client = await getClient(); + const transformId = args.transform === undefined ? undefined : parseId(args.transform); + const { data } = await client.transformTest.list({ "transform-id": transformId }); + renderList(windowList(data, ctx.range), transformTestView, ctx); + }, +}); diff --git a/packages/cli/src/commands/transform-test/run.ts b/packages/cli/src/commands/transform-test/run.ts new file mode 100644 index 00000000..f82a3ba6 --- /dev/null +++ b/packages/cli/src/commands/transform-test/run.ts @@ -0,0 +1,61 @@ +import type { + TransformTestExpectationResult, + TransformTestRunResult, + TransformTestStatus, +} from "@metabase/client/domain/transform-test"; +import { TransformTestRunResult as TransformTestRunResultSchema } from "@metabase/client/domain/transform-test"; + +import { renderSummary } from "../../output/render"; +import type { ResourceView } from "../../output/view"; +import { connectionFlags, outputFlags, profileFlag } from "../flags"; +import { parseId } from "../parse-id"; +import { defineMetabaseCommand } from "../runtime"; + +const transformTestRunResultView: ResourceView = { + compactPick: TransformTestRunResultSchema, + tableColumns: [{ key: "status", label: "Status" }], +}; + +function namesOf( + expectations: TransformTestExpectationResult[], + status: TransformTestStatus, +): string[] { + return expectations.filter((expectation) => expectation.status === status).map((e) => e.name); +} + +function summaryLine(id: number, result: TransformTestRunResult): string { + if (result.status === "passed") { + return `Transform test ${id} passed ${result.expectations.length} expectation(s).`; + } + const failed = namesOf(result.expectations, "failed"); + const errored = namesOf(result.expectations, "error"); + const parts = [ + failed.length > 0 ? `failed: ${failed.join(", ")}` : null, + errored.length > 0 ? `could not run: ${errored.join(", ")}` : null, + ].filter((part) => part !== null); + return `Transform test ${id} ${result.status} — ${parts.join("; ")}.`; +} + +export default defineMetabaseCommand({ + meta: { name: "run", description: "Run a transform test by id" }, + details: + "Runs the transform against temp tables built from the test's inputs, checks every expectation against its output, and drops the temp tables. Nothing reads or writes a real table. A failing expectation exits non-zero; --json reports what each expectation found, including the rows a comparison disagreed on.", + capabilities: { minVersion: 64 }, + args: { + ...outputFlags, + ...profileFlag, + ...connectionFlags, + id: { type: "positional", description: "Transform test id", required: true }, + }, + outputSchema: TransformTestRunResultSchema, + examples: ["mb transform-test run 1", "mb transform-test run 1 --json"], + async run({ args, ctx, getClient }) { + const id = parseId(args.id); + const client = await getClient(); + const result = await client.transformTest.run(id); + renderSummary(result, transformTestRunResultView, summaryLine(id, result), ctx); + if (result.status !== "passed") { + throw new Error(`transform test ${id} ${result.status}`); + } + }, +}); diff --git a/packages/cli/src/commands/transform-test/update.ts b/packages/cli/src/commands/transform-test/update.ts new file mode 100644 index 00000000..9116b969 --- /dev/null +++ b/packages/cli/src/commands/transform-test/update.ts @@ -0,0 +1,36 @@ +import { TransformTest, TransformTestUpdateInput } from "@metabase/client/domain/transform-test"; + +import { renderSummary } from "../../output/render"; +import { transformTestView } from "../../output/views/transform-test"; +import { readBody } from "../../runtime/body"; +import { bodyInputFlags } from "../body-flags"; +import { connectionFlags, outputFlags, profileFlag } from "../flags"; +import { parseId } from "../parse-id"; +import { defineMetabaseCommand } from "../runtime"; + +export default defineMetabaseCommand({ + meta: { name: "update", description: "Update a transform test from JSON" }, + details: + "Only the fields the body carries are patched. `inputs` and `expectations` replace what is stored rather than merging into it.", + capabilities: { minVersion: 64 }, + args: { + ...outputFlags, + ...profileFlag, + ...connectionFlags, + ...bodyInputFlags, + id: { type: "positional", description: "Transform test id", required: true }, + }, + inputSchema: TransformTestUpdateInput, + outputSchema: TransformTest, + examples: [ + "cat patch.json | mb transform-test update 1", + "mb transform-test update 1 --file patch.json", + ], + async run({ args, ctx, getClient }) { + const id = parseId(args.id); + const body = await readBody({ flag: args.body, file: args.file }, TransformTestUpdateInput); + const client = await getClient(); + const updated = await client.transformTest.update(id, body); + renderSummary(updated, transformTestView, `Updated transform test ${updated.id}.`, ctx); + }, +}); diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 420a0ace..1bb40d1c 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -29,6 +29,7 @@ const main: CommandDef = defineCommand({ transform: () => import("./commands/transform").then((mod) => mod.default), "transform-job": () => import("./commands/transform-job").then((mod) => mod.default), "transform-tag": () => import("./commands/transform-tag").then((mod) => mod.default), + "transform-test": () => import("./commands/transform-test").then((mod) => mod.default), setting: () => import("./commands/setting").then((mod) => mod.default), search: () => import("./commands/search").then((mod) => mod.default), "git-sync": () => import("./commands/git-sync").then((mod) => mod.default), diff --git a/packages/cli/src/output/views/transform-test.ts b/packages/cli/src/output/views/transform-test.ts new file mode 100644 index 00000000..a194f851 --- /dev/null +++ b/packages/cli/src/output/views/transform-test.ts @@ -0,0 +1,13 @@ +import { type TransformTest, TransformTestCompact } from "@metabase/client/domain/transform-test"; + +import type { ResourceView } from "../view"; + +export const transformTestView: ResourceView = { + compactPick: TransformTestCompact, + tableColumns: [ + { key: "id", label: "ID" }, + { key: "transform_id", label: "Transform" }, + { key: "name", label: "Name" }, + { key: "description", label: "Description" }, + ], +}; diff --git a/packages/cli/src/runtime/command-help.test.ts b/packages/cli/src/runtime/command-help.test.ts index aac33b4f..e802cbc5 100644 --- a/packages/cli/src/runtime/command-help.test.ts +++ b/packages/cli/src/runtime/command-help.test.ts @@ -390,6 +390,12 @@ const ALL_COMMANDS = [ "transform-tag create", "transform-tag update", "transform-tag delete", + "transform-test list", + "transform-test get", + "transform-test create", + "transform-test update", + "transform-test delete", + "transform-test run", "setting list", "setting get", "setting set", @@ -452,6 +458,7 @@ const CONTENT_TRANSLATION_CAPABILITIES = { } as const; const TRANSFORM_CAPABILITIES = { minVersion: 59 } as const; const TRANSFORM_JOB_SET_ACTIVE_CAPABILITIES = { minVersion: 61 } as const; +const TRANSFORM_TEST_CAPABILITIES = { minVersion: 64 } as const; let cachedEntries: Promise | null = null; @@ -558,6 +565,23 @@ describe("command tree contract", () => { }); }); + it("gates every transform-test command at the version that first serves the endpoint", async () => { + const entries = await allEntries(); + const capabilities = Object.fromEntries( + entries + .filter((entry) => entry.command.startsWith("transform-test ")) + .map((entry) => [entry.command, entry.capabilities]), + ); + expect(capabilities).toEqual({ + "transform-test list": TRANSFORM_TEST_CAPABILITIES, + "transform-test get": TRANSFORM_TEST_CAPABILITIES, + "transform-test create": TRANSFORM_TEST_CAPABILITIES, + "transform-test update": TRANSFORM_TEST_CAPABILITIES, + "transform-test delete": TRANSFORM_TEST_CAPABILITIES, + "transform-test run": TRANSFORM_TEST_CAPABILITIES, + }); + }); + it("advertises a --limit default on search alone, so the shared flag description stays true", async () => { const entries = await allEntries(); const withDefault = Object.fromEntries( diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index 4ad45d80..39cdf0bf 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -24,6 +24,7 @@ import { timelineResource } from "./resources/timeline"; import { transformJobResource } from "./resources/transform-job"; import { transformResource } from "./resources/transform"; import { transformTagResource } from "./resources/transform-tag"; +import { transformTestResource } from "./resources/transform-test"; import { uploadResource } from "./resources/upload"; import { userResource } from "./resources/user"; @@ -58,6 +59,7 @@ export function createClient(config: ClientCredentials, options: ClientOptions) transform: transformResource(transport), transformJob: transformJobResource(transport), transformTag: transformTagResource(transport), + transformTest: transformTestResource(transport), upload: uploadResource(transport), user: userResource(transport), // The escape hatch: an endpoint the client never models still has to cost a consumer nothing. diff --git a/packages/client/src/domain/transform-test.ts b/packages/client/src/domain/transform-test.ts new file mode 100644 index 00000000..d16b4a00 --- /dev/null +++ b/packages/client/src/domain/transform-test.ts @@ -0,0 +1,170 @@ +import { z } from "zod"; + +const TransformTestTable = z + .object({ + schema: z.string().nullable().optional(), + name: z.string().min(1), + }) + .loose(); + +const TransformTestColumn = z + .object({ + name: z.string().min(1), + database_type: z.string().min(1).describe("The warehouse's own spelling, e.g. VARCHAR(255)"), + }) + .loose(); + +const TransformTestCell = z.union([z.string(), z.number(), z.boolean(), z.null()]); + +const TransformTestRow = z.record(z.string(), TransformTestCell); + +const TransformTestSqlData = z.object({ format: z.literal("sql"), sql: z.string().min(1) }).loose(); + +const TransformTestRowsData = z + .object({ + format: z.literal("rows"), + columns: z.array(TransformTestColumn).min(1), + rows: z.array(TransformTestRow), + }) + .loose(); + +const TransformTestSqlInput = TransformTestSqlData.extend({ table: TransformTestTable }); + +const TransformTestRowsInput = TransformTestRowsData.extend({ table: TransformTestTable }); + +/** Test data standing in for one table the transform reads. */ +export const TransformTestInput = z.discriminatedUnion("format", [ + TransformTestSqlInput, + TransformTestRowsInput, +]); +export type TransformTestInput = z.infer; + +const TransformTestEqualsSql = TransformTestSqlData.extend({ + type: z.literal("equals"), + name: z.string().min(1), +}); + +const TransformTestEqualsRows = TransformTestRowsData.extend({ + type: z.literal("equals"), + name: z.string().min(1), +}); + +const TransformTestEmpty = z + .object({ + type: z.literal("empty"), + name: z.string().min(1), + sql: z.string().min(1), + }) + .loose(); + +/** One check against the transform's output. Names are unique within a test. */ +export const TransformTestExpectation = z.union([ + TransformTestEqualsSql, + TransformTestEqualsRows, + TransformTestEmpty, +]); +export type TransformTestExpectation = z.infer; + +export const TransformTest = z + .object({ + id: z.number().int(), + entity_id: z.string().nullable(), + transform_id: z.number().int(), + creator_id: z.number().int(), + name: z.string(), + description: z.string().nullable(), + inputs: z.array(TransformTestInput), + expectations: z.array(TransformTestExpectation), + created_at: z.string(), + updated_at: z.string(), + }) + .loose(); +export type TransformTest = z.infer; + +export const TransformTestCompact = TransformTest.pick({ + id: true, + transform_id: true, + name: true, + description: true, +}).strip(); +export type TransformTestCompact = z.infer; + +export const TransformTestCreateInput = z + .object({ + transform_id: z.number().int().positive(), + name: z.string().min(1), + description: z.string().nullable().optional(), + inputs: z.array(TransformTestInput), + expectations: z.array(TransformTestExpectation), + }) + .loose(); +export type TransformTestCreateInput = z.infer; + +export const TransformTestUpdateInput = z + .object({ + transform_id: z.number().int().positive().optional(), + name: z.string().min(1).optional(), + description: z.string().nullable().optional(), + inputs: z.array(TransformTestInput).optional(), + expectations: z.array(TransformTestExpectation).optional(), + }) + .loose(); +export type TransformTestUpdateInput = z.infer; + +/** + * `error` is one expectation's own failure to run, which leaves the rest of the run reporting; the + * run as a whole is only `passed` or `failed`. + */ +export const TransformTestStatus = z.enum(["passed", "failed", "error"]); +export type TransformTestStatus = z.infer; + +const TransformTestResultColumn = z.object({ name: z.string(), database_type: z.string() }).loose(); + +const TransformTestCellMismatch = z + .object({ + column: z.string(), + expected: TransformTestCell, + actual: TransformTestCell, + }) + .loose(); + +const TransformTestRowCounts = z + .object({ actual: z.number().int(), expected: z.number().int() }) + .loose(); + +const TransformTestExpectationError = z.object({ type: z.string(), message: z.string() }).loose(); + +/** + * What one expectation found. The keys beyond `name`, `type` and `status` are its type's own: + * `equals` reports the rows the two sides disagree on, `empty` a sample of the rows its query + * returned, and both cap what they report, with `truncated` counting what the cap dropped. + */ +export const TransformTestExpectationResult = z + .object({ + name: z.string(), + type: z.enum(["equals", "empty"]), + status: TransformTestStatus, + columns: z.array(TransformTestResultColumn).optional(), + error: TransformTestExpectationError.optional(), + "row-counts": TransformTestRowCounts.optional(), + "extra-rows": z.array(TransformTestRow).optional(), + "missing-rows": z.array(TransformTestRow).optional(), + "cell-mismatches": z.array(TransformTestCellMismatch).optional(), + sample: z.array(TransformTestRow).optional(), + truncated: z.number().int().optional(), + }) + .loose(); +export type TransformTestExpectationResult = z.infer; + +/** + * What a run reported. `tables` maps each temp table the run created to the table it stood in for. + * A run that happened reports `passed` or `failed`; a refusal is an HTTP error rather than a body. + */ +export const TransformTestRunResult = z + .object({ + status: TransformTestStatus, + expectations: z.array(TransformTestExpectationResult), + tables: z.record(z.string(), z.string()), + }) + .loose(); +export type TransformTestRunResult = z.infer; diff --git a/packages/client/src/index.test.ts b/packages/client/src/index.test.ts index 7a46af25..6ab2e73d 100644 --- a/packages/client/src/index.test.ts +++ b/packages/client/src/index.test.ts @@ -42,6 +42,9 @@ import { TransformJobUpdateInput, TransformTagCreateInput, TransformTagUpdateInput, + TransformTestCreateInput, + TransformTestInput, + TransformTestUpdateInput, TransformUpdateInput, } from "./index"; import type { @@ -105,6 +108,9 @@ const writePath = { TransformJobUpdateInput: pin(TransformJobUpdateInput), TransformTagCreateInput: pin(TransformTagCreateInput), TransformTagUpdateInput: pin(TransformTagUpdateInput), + TransformTestCreateInput: pin(TransformTestCreateInput), + TransformTestInput: pin(TransformTestInput), + TransformTestUpdateInput: pin(TransformTestUpdateInput), TransformUpdateInput: pin(TransformUpdateInput), }; diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 356dcb0a..ccddefeb 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -272,5 +272,16 @@ export { TransformTagCreateInput, TransformTagUpdateInput, } from "./domain/transform-tag"; +export { + TransformTest, + TransformTestCompact, + TransformTestCreateInput, + TransformTestExpectation, + TransformTestExpectationResult, + TransformTestInput, + TransformTestRunResult, + TransformTestStatus, + TransformTestUpdateInput, +} from "./domain/transform-test"; export { UploadResult, UploadUpdateAction, UploadUpdateResult } from "./domain/upload"; export { CurrentUser, CurrentUserCompact } from "./domain/user"; diff --git a/packages/client/src/resources/transform-test.test.ts b/packages/client/src/resources/transform-test.test.ts new file mode 100644 index 00000000..7090bdb3 --- /dev/null +++ b/packages/client/src/resources/transform-test.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it } from "vitest"; + +import { createClient } from "../client"; +import type { ClientCredentials } from "../http/transport"; +import { + captureFetch, + type FetchScript, + jsonResponse, + TEST_USER_AGENT, +} from "../testing/fetch-capture"; + +const CREDENTIALS: ClientCredentials = { + url: "https://mb.example.com/metabase", + credential: { kind: "apiKey", apiKey: "mb_wire_test_key" }, +}; + +const TRANSFORM_TEST = { + id: 3, + entity_id: "AbCdEfGhIjKlMnOpQrStU", + transform_id: 7, + creator_id: 1, + name: "people summary holds one row", + description: null, + inputs: [ + { + table: { schema: "public", name: "people" }, + format: "rows", + columns: [{ name: "id", database_type: "INTEGER" }], + rows: [{ id: 1 }], + }, + ], + expectations: [{ type: "empty", name: "no null ids", sql: "SELECT * FROM out WHERE id IS NULL" }], + created_at: "2026-09-15T12:00:00Z", + updated_at: "2026-09-15T12:00:00Z", +}; + +const RUN_RESULT = { + status: "failed", + expectations: [ + { + name: "no null ids", + type: "empty", + status: "failed", + columns: [{ name: "id", database_type: "INTEGER" }], + sample: [{ id: null }], + truncated: 0, + }, + ], + tables: { mb_test_a1b2: "public.people" }, +}; + +const JSON_REQUEST_HEADERS = { + accept: "application/json", + "content-type": "application/json", + "user-agent": TEST_USER_AGENT, + "x-api-key": "mb_wire_test_key", +}; + +const JSON_READ_HEADERS = { + accept: "application/json", + "user-agent": TEST_USER_AGENT, + "x-api-key": "mb_wire_test_key", +}; + +const BINARY_READ_HEADERS = { + accept: "*/*", + "user-agent": TEST_USER_AGENT, + "x-api-key": "mb_wire_test_key", +}; + +function clientOver(responses: FetchScript) { + const capture = captureFetch(responses); + const mb = createClient(CREDENTIALS, { + userAgent: TEST_USER_AGENT, + fetchImpl: capture.fetch, + }); + return { mb, capture }; +} + +describe("transform-test resource wire requests", () => { + it("sends the list request with no query string when no transform is named", async () => { + const { mb, capture } = clientOver([jsonResponse([TRANSFORM_TEST])]); + + await mb.transformTest.list(); + + expect(capture.calls).toEqual([ + { + url: "https://mb.example.com/metabase/api/transform-test", + method: "GET", + headers: JSON_READ_HEADERS, + body: null, + }, + ]); + }); + + it("sends the list request narrowed to one transform", async () => { + const { mb, capture } = clientOver([jsonResponse([TRANSFORM_TEST])]); + + await mb.transformTest.list({ "transform-id": 7 }); + + expect(capture.calls).toEqual([ + { + url: "https://mb.example.com/metabase/api/transform-test?transform-id=7", + method: "GET", + headers: JSON_READ_HEADERS, + body: null, + }, + ]); + }); + + it("reports no total for the listing, which the server does not count", async () => { + const { mb } = clientOver([jsonResponse([TRANSFORM_TEST])]); + + expect(await mb.transformTest.list()).toEqual({ data: [TRANSFORM_TEST], total: null }); + }); + + it("sends the get request", async () => { + const { mb, capture } = clientOver([jsonResponse(TRANSFORM_TEST)]); + + await mb.transformTest.get(3); + + expect(capture.calls).toEqual([ + { + url: "https://mb.example.com/metabase/api/transform-test/3", + method: "GET", + headers: JSON_READ_HEADERS, + body: null, + }, + ]); + }); + + it("sends the create request with the body it was given", async () => { + const { mb, capture } = clientOver([jsonResponse(TRANSFORM_TEST)]); + + await mb.transformTest.create({ + transform_id: 7, + name: TRANSFORM_TEST.name, + inputs: [ + { + table: { schema: "public", name: "people" }, + format: "sql", + sql: "SELECT 1 AS id", + }, + ], + expectations: [{ type: "empty", name: "no null ids", sql: "SELECT * FROM out" }], + }); + + expect(capture.calls).toEqual([ + { + url: "https://mb.example.com/metabase/api/transform-test", + method: "POST", + headers: JSON_REQUEST_HEADERS, + body: '{"transform_id":7,"name":"people summary holds one row","inputs":[{"table":{"schema":"public","name":"people"},"format":"sql","sql":"SELECT 1 AS id"}],"expectations":[{"type":"empty","name":"no null ids","sql":"SELECT * FROM out"}]}', + }, + ]); + }); + + it("sends the update request as a PUT carrying only the fields it patches", async () => { + const { mb, capture } = clientOver([jsonResponse({ ...TRANSFORM_TEST, name: "renamed" })]); + + await mb.transformTest.update(3, { name: "renamed" }); + + expect(capture.calls).toEqual([ + { + url: "https://mb.example.com/metabase/api/transform-test/3", + method: "PUT", + headers: JSON_REQUEST_HEADERS, + body: '{"name":"renamed"}', + }, + ]); + }); + + it("sends the delete request as a bodiless DELETE accepting any content type", async () => { + const { mb, capture } = clientOver([new Response(null, { status: 204 })]); + + await mb.transformTest.delete(3); + + expect(capture.calls).toEqual([ + { + url: "https://mb.example.com/metabase/api/transform-test/3", + method: "DELETE", + headers: BINARY_READ_HEADERS, + body: null, + }, + ]); + }); + + it("sends the run request as a bodiless POST", async () => { + const { mb, capture } = clientOver([jsonResponse(RUN_RESULT)]); + + await mb.transformTest.run(3); + + expect(capture.calls).toEqual([ + { + url: "https://mb.example.com/metabase/api/transform-test/3/run", + method: "POST", + headers: JSON_READ_HEADERS, + body: null, + }, + ]); + }); + + it("returns what each expectation found, keeping the wire's kebab-case report keys", async () => { + const { mb } = clientOver([ + jsonResponse({ + ...RUN_RESULT, + expectations: [ + { + name: "output", + type: "equals", + status: "failed", + "row-counts": { actual: 2, expected: 1 }, + "extra-rows": [{ id: 1 }], + "missing-rows": [], + "cell-mismatches": [{ column: "id", expected: 1, actual: 2 }], + truncated: 0, + }, + ], + }), + ]); + + expect(await mb.transformTest.run(3)).toEqual({ + status: "failed", + expectations: [ + { + name: "output", + type: "equals", + status: "failed", + "row-counts": { actual: 2, expected: 1 }, + "extra-rows": [{ id: 1 }], + "missing-rows": [], + "cell-mismatches": [{ column: "id", expected: 1, actual: 2 }], + truncated: 0, + }, + ], + tables: { mb_test_a1b2: "public.people" }, + }); + }); +}); diff --git a/packages/client/src/resources/transform-test.ts b/packages/client/src/resources/transform-test.ts new file mode 100644 index 00000000..b93d3a52 --- /dev/null +++ b/packages/client/src/resources/transform-test.ts @@ -0,0 +1,86 @@ +import { z } from "zod"; + +import { + TransformTest, + type TransformTestCreateInput, + TransformTestRunResult, + type TransformTestUpdateInput, +} from "../domain/transform-test"; +import type { RequestOptions, Transport } from "../http/transport"; +import type { ListResult } from "../list"; + +// `GET /api/transform-test` answers a bare array rather than a `{ data, total }` envelope, so the +// count a caller reads off `ListResult` is the array's own length and the server reports none. +const TransformTestApiList = z.array(TransformTest); + +export interface TransformTestListParams { + "transform-id"?: number | undefined; +} + +// Every path parameter here is a numeric id, so no fragment needs `encodeURIComponent`. +export function transformTestResource(transport: Transport) { + /** List the transform tests the caller can see, optionally only those of one transform. */ + async function list( + params: TransformTestListParams = {}, + options: RequestOptions = {}, + ): Promise> { + const data = await transport.requestParsed(TransformTestApiList, "/api/transform-test", { + ...options, + query: { "transform-id": params["transform-id"] }, + }); + return { data, total: null }; + } + + /** Get one transform test by id. */ + async function get(id: number, options: RequestOptions = {}): Promise { + return transport.requestParsed(TransformTest, `/api/transform-test/${id}`, { ...options }); + } + + /** Create a transform test — the inputs standing in for the transform's source tables, and the expectations checked against its output. */ + async function create( + params: TransformTestCreateInput, + options: RequestOptions = {}, + ): Promise { + return transport.requestParsed(TransformTest, "/api/transform-test", { + ...options, + method: "POST", + body: params, + }); + } + + /** Update a transform test by id, patching only the fields the body carries. */ + async function update( + id: number, + params: TransformTestUpdateInput, + options: RequestOptions = {}, + ): Promise { + return transport.requestParsed(TransformTest, `/api/transform-test/${id}`, { + ...options, + method: "PUT", + body: params, + }); + } + + /** Delete a transform test by id. */ + async function remove(id: number, options: RequestOptions = {}): Promise { + await transport.requestRaw(`/api/transform-test/${id}`, { + ...options, + method: "DELETE", + expectContentType: "binary", + }); + } + + /** + * Run a transform test against temp tables and report what each expectation found. The transform + * itself never runs against real tables, and a failing expectation is a result rather than an + * error. + */ + async function run(id: number, options: RequestOptions = {}): Promise { + return transport.requestParsed(TransformTestRunResult, `/api/transform-test/${id}/run`, { + ...options, + method: "POST", + }); + } + + return { list, get, create, update, delete: remove, run }; +} From efc67955520a6df86f0e7db928624fd9344b341f Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Tue, 15 Sep 2026 23:39:31 -0400 Subject: [PATCH 2/8] Close the transform-test bodies and document the shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The server closes `::transform-test.create` and `::transform-test.update`, so a body cloned from `get` earned an opaque 400. Closing them here names the offending key instead; a stored test needs `id`, `entity_id`, `creator_id`, `created_at` and `updated_at` stripped before it can be sent back, which the command help, the README and the transform skill now say. `get` projects through the compact view, so `--full` is what shows `inputs` and `expectations` — the examples and docs advertise it. An `equals` expectation needs `format: "sql"` or `format: "rows"`, which the docs omitted, leaving a body written from them to fail on a three-branch union error. Co-Authored-By: Claude Opus 5 --- README.md | 17 +++- packages/cli/skill-data/core/SKILL.md | 2 +- packages/cli/skill-data/transform/SKILL.md | 51 +++++++++++- .../cli/src/commands/transform-test/create.ts | 2 +- .../cli/src/commands/transform-test/get.ts | 4 +- .../cli/src/commands/transform-test/update.ts | 2 +- .../client/src/domain/transform-test.test.ts | 80 +++++++++++++++++++ packages/client/src/domain/transform-test.ts | 10 ++- 8 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 packages/client/src/domain/transform-test.test.ts diff --git a/README.md b/README.md index 5fabe33c..e1954a1e 100644 --- a/README.md +++ b/README.md @@ -383,7 +383,17 @@ mb transform-tag delete 5 --yes CRUD and run on `/api/transform-test`. Requires Metabase v64 or newer. A transform test pins a transform's behaviour without touching real data: every table the transform reads is replaced by an `input` fixture, the transform runs into a temp table, and each `expectation` checks that output. The temp tables are dropped when the run ends. -An input names its `table` and carries either `format: "sql"` with a `sql` query or `format: "rows"` with `columns` (each with a `database_type` the warehouse accepts as a `CAST` target) and `rows`. An expectation is either `type: "empty"` with the `sql` that must return no rows, or `type: "equals"` with the rows the output must hold exactly. +An input names its `table` and carries either `format: "sql"` with a `sql` query or `format: "rows"` with `columns` (each with a `database_type` the warehouse accepts as a `CAST` target) and `rows`. + +An expectation is either `type: "empty"` with the `sql` that must return no rows, or `type: "equals"`, which takes the same `format` split as an input — `format: "rows"` with the `columns` and `rows` the output must hold exactly, or `format: "sql"` with a query returning them. An `equals` without a `format` is refused. + +Create and update bodies are closed, so a test read back with `get --full` has to shed `id`, `entity_id`, `creator_id`, `created_at` and `updated_at` before it can be sent back: + +```sh +mb transform-test get 1 --full --json \ + | jq 'del(.id, .entity_id, .creator_id, .created_at, .updated_at)' \ + | mb transform-test update 1 +``` ### `mb transform-test list` @@ -398,8 +408,11 @@ mb transform-test list --transform 1 ### `mb transform-test get ` +The compact form carries the id, transform, name and description; `--full` adds the `inputs` and `expectations` themselves. + ```sh -mb transform-test get 1 --json +mb transform-test get 1 +mb transform-test get 1 --full --json ``` ### `mb transform-test create` diff --git a/packages/cli/skill-data/core/SKILL.md b/packages/cli/skill-data/core/SKILL.md index 6d30d1cb..db8fb953 100644 --- a/packages/cli/skill-data/core/SKILL.md +++ b/packages/cli/skill-data/core/SKILL.md @@ -140,7 +140,7 @@ Routine verb shapes (list / get / create / update), every flag, and output schem - **collection ``** accepts four forms only — positive int, `root`, `trash`, or a 21-char entity_id; anything else is a client-side `ConfigError`. `collection items` pages the server endpoint, pulling only as far as the output cap can show — read `has_more`/`next_offset` to continue. `collection tree` is **JSON-only** (`--format text` is rejected). A transform collection needs `collection create --namespace transforms`. - **setting set** parses the value as **strict JSON**: a string is `'"value"'` (inner quotes), booleans `true`/`false`, numbers bare. Wrong quoting silently errors — confirm with `setting get ` after. `setting get --json` works on every value type (wrapping bare-text responses into `{key, value}`). - **search vs. list.** For plain enumeration of cards/dashboards/collections use the dedicated `… list` verbs; reach for `search --models ` only for ranking against a query string or a cross-resource lookup. -- **transform.** Iterate with `transform update `, never `delete` + `create` (keeps the row, `entity_id`, materialized table, and YAML filename — avoids `_2` suffixes and noisy git history). `transform run` needs `--wait` (or `--sync`, which also waits for the output table to register and returns `target_table_id`) or you get only `{run_id, final:null}`. (→ `transform`.) +- **transform.** Iterate with `transform update `, never `delete` + `create` (keeps the row, `entity_id`, materialized table, and YAML filename — avoids `_2` suffixes and noisy git history). `transform run` needs `--wait` (or `--sync`, which also waits for the output table to register and returns `target_table_id`) or you get only `{run_id, final:null}`. `transform-test run ` checks a transform against fixtures in temp tables — no real table is read — and exits non-zero unless it passes (v64+). (→ `transform`.) - **setup is one-shot.** `mb setup` walks `/api/setup` for a **fresh** instance only — errors against an already-configured one. Mostly for bootstrapping local / e2e instances. - **eid** translates a string entity id → numeric id: `mb eid --model --json`. Entity ids are NanoIDs that can start with `-`, which the positional form misreads as a flag (shell quotes don't help) — for those, use `--body '{"entity_ids":{"card":["-…"]}}'` (the id is a JSON string value, immune to flag parsing). - **library.** EE-only (`library` premium feature, v59+). The Library is a curated subtree (`library-data` "Data" + `library-metrics` "Metrics" under a `library` root): tables published to **Data** appear first in data pickers and rank up in search; metrics saved to **Metrics** are prioritized in nav, search, and the query builder — it's how you tell people (and agents) "start from these, they're trusted." `library get` shows the Library and its Data/Metrics collection ids; `library create` provisions it (idempotent). `library publish --table-ids/--db-ids/--schemas` publishes tables into Data — it **resolves the Data collection itself and creates the Library if absent** (no collection id to find); each `--schemas` entry is `:` (e.g. `1:public`), not a bare name. `publish` cascades to upstream FK dependencies, `unpublish` to downstream dependents; both need **admin or data-analyst** (Curate alone won't publish) and exit **403** without write **and** query permission on every affected table. Publish status shows on the table: `table get`/`table list` carry `is_published` (`collection_id` under `--full`). Good candidates are finished, analysis-ready tables — clean/combine via transforms first, then publish the polished result. Publishing does not put the Data collection in the git-sync scope: on a remote-sync instance, `mb git-sync add-collection ` is what makes exports carry the published tables' metadata (see the `git-sync` skill). diff --git a/packages/cli/skill-data/transform/SKILL.md b/packages/cli/skill-data/transform/SKILL.md index b7fd015c..e5fbaa15 100644 --- a/packages/cli/skill-data/transform/SKILL.md +++ b/packages/cli/skill-data/transform/SKILL.md @@ -1,6 +1,6 @@ --- name: transform -description: Author and run Metabase transforms via `mb` — body shape (native SQL or structured MBQL), create + run-with-wait, run inspection, dependencies, cancel, the `update`-vs-recreate iteration rule, the writable-keys-only PATCH contract, plus transform tags and tag-driven transform-job schedules. Load when the user touches transforms — "create a transform", "run a transform", "fix a failing transform", "list transform runs", "cancel a running transform", "manage transform tags", "run a transform job", or anything `mb transform …` / `mb transform-job …` / `mb transform-tag …`. +description: Author and run Metabase transforms via `mb` — body shape (native SQL or structured MBQL), create + run-with-wait, run inspection, dependencies, cancel, the `update`-vs-recreate iteration rule, the writable-keys-only PATCH contract, transform tests (fixtures and expectations run against temp tables), plus transform tags and tag-driven transform-job schedules. Load when the user touches transforms — "create a transform", "run a transform", "fix a failing transform", "test a transform", "list transform runs", "cancel a running transform", "manage transform tags", "run a transform job", or anything `mb transform …` / `mb transform-job …` / `mb transform-tag …` / `mb transform-test …`. allowed-tools: Read, Write, Edit, Bash, AskUserQuestion --- @@ -172,6 +172,55 @@ mb transform run "$ID" --wait --profile --json # → succeeded If you really must `create + delete` instead, do the `delete` **before** the first `git-sync export` so the failed entity never lands in git history — an export of a soft-failed state is noise that needs a follow-up cleanup commit. See `git-sync`, "Read state before mutating", for the ordering rule. +## Transform tests (v64+) + +A transform test replaces every table the transform reads with a fixture, runs it into a temp table, and checks that output. No real table is read or written. + +```bash +mb transform-test list --transform --profile --json # --transform is optional +mb transform-test get --full --profile --json # --full for inputs/expectations +mb transform-test create --file ./.scratch/test.json --profile --json +mb transform-test update --file ./.scratch/patch.json --profile --json +mb transform-test delete --yes --profile +mb transform-test run --profile --json # exits non-zero unless it passes +``` + +**`inputs`** — one per table the transform reads, each naming a `table` plus either `format: "sql"` with `sql`, or `format: "rows"` with `columns` (each a `name` and a `database_type` the warehouse takes as a `CAST` target) and `rows`. + +**`expectations`** — `type: "empty"` with the `sql` that must return no rows, or `type: "equals"`, which needs the same `format` split as an input (`"rows"` with `columns`/`rows`, or `"sql"` with a query). An `equals` without a `format` is refused. + +```json +{ + "transform_id": 1, + "name": "adults only", + "inputs": [ + { + "table": { "schema": "public", "name": "people" }, + "format": "rows", + "columns": [ + { "name": "id", "database_type": "INTEGER" }, + { "name": "age", "database_type": "INTEGER" } + ], + "rows": [ + { "id": 1, "age": 30 }, + { "id": 2, "age": 12 } + ] + } + ], + "expectations": [ + { + "type": "equals", + "name": "keeps only the adult", + "format": "rows", + "columns": [{ "name": "id", "database_type": "INTEGER" }], + "rows": [{ "id": 1 }] + } + ] +} +``` + +Create and update bodies are closed — strip `id`, `entity_id`, `creator_id`, `created_at` and `updated_at` from a `get --full` body before sending it back. + ## Drop the materialized table (keep the transform) ```bash diff --git a/packages/cli/src/commands/transform-test/create.ts b/packages/cli/src/commands/transform-test/create.ts index 50487782..d1a4bfaf 100644 --- a/packages/cli/src/commands/transform-test/create.ts +++ b/packages/cli/src/commands/transform-test/create.ts @@ -10,7 +10,7 @@ import { defineMetabaseCommand } from "../runtime"; export default defineMetabaseCommand({ meta: { name: "create", description: "Create a transform test from JSON" }, details: - 'The JSON body needs a `transform_id`, a `name`, the `inputs` standing in for every table the transform reads, and the `expectations` checked against its output. An input names its `table` and carries either `format: "sql"` with a `sql` query or `format: "rows"` with `columns` and `rows`. An expectation is either `type: "empty"` with the `sql` that must return nothing, or `type: "equals"` with the rows the output must hold.', + 'The JSON body needs a `transform_id`, a `name`, the `inputs` standing in for every table the transform reads, and the `expectations` checked against its output. An input names its `table` and carries either `format: "sql"` with a `sql` query or `format: "rows"` with `columns` and `rows`. An expectation is either `type: "empty"` with the `sql` that must return nothing, or `type: "equals"`, which takes the same `format` split as an input: `format: "rows"` with the `columns` and `rows` the output must hold, or `format: "sql"` with a query returning them. The body is closed: a test read back with `get --full` has to shed `id`, `entity_id`, `creator_id`, `created_at` and `updated_at` before it can be sent.', capabilities: { minVersion: 64 }, args: { ...outputFlags, diff --git a/packages/cli/src/commands/transform-test/get.ts b/packages/cli/src/commands/transform-test/get.ts index 66e46c96..b3872fca 100644 --- a/packages/cli/src/commands/transform-test/get.ts +++ b/packages/cli/src/commands/transform-test/get.ts @@ -8,6 +8,8 @@ import { defineMetabaseCommand } from "../runtime"; export default defineMetabaseCommand({ meta: { name: "get", description: "Get a transform test by id" }, + details: + "The compact form carries the id, transform, name and description. Pass --full for the `inputs` and `expectations` themselves, which is also the body to edit and send back to `update`.", capabilities: { minVersion: 64 }, args: { ...outputFlags, @@ -16,7 +18,7 @@ export default defineMetabaseCommand({ id: { type: "positional", description: "Transform test id", required: true }, }, outputSchema: TransformTest, - examples: ["mb transform-test get 1", "mb transform-test get 1 --json"], + examples: ["mb transform-test get 1", "mb transform-test get 1 --full --json"], async run({ args, ctx, getClient }) { const id = parseId(args.id); const client = await getClient(); diff --git a/packages/cli/src/commands/transform-test/update.ts b/packages/cli/src/commands/transform-test/update.ts index 9116b969..7b5699dd 100644 --- a/packages/cli/src/commands/transform-test/update.ts +++ b/packages/cli/src/commands/transform-test/update.ts @@ -11,7 +11,7 @@ import { defineMetabaseCommand } from "../runtime"; export default defineMetabaseCommand({ meta: { name: "update", description: "Update a transform test from JSON" }, details: - "Only the fields the body carries are patched. `inputs` and `expectations` replace what is stored rather than merging into it.", + "Only the fields the body carries are patched. `inputs` and `expectations` replace what is stored rather than merging into it. The body is closed: a test read back with `get --full` has to shed `id`, `entity_id`, `creator_id`, `created_at` and `updated_at` before it can be sent.", capabilities: { minVersion: 64 }, args: { ...outputFlags, diff --git a/packages/client/src/domain/transform-test.test.ts b/packages/client/src/domain/transform-test.test.ts new file mode 100644 index 00000000..025a811e --- /dev/null +++ b/packages/client/src/domain/transform-test.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; + +import { + TransformTest, + TransformTestCreateInput, + TransformTestUpdateInput, +} from "./transform-test"; + +const STORED = { + id: 3, + entity_id: "AbCdEfGhIjKlMnOpQrStU", + transform_id: 7, + creator_id: 1, + name: "people summary holds one row", + description: null, + inputs: [{ table: { schema: "public", name: "people" }, format: "sql", sql: "SELECT 1 AS id" }], + expectations: [{ type: "empty", name: "no null ids", sql: "SELECT * FROM out" }], + created_at: "2026-09-15T12:00:00Z", + updated_at: "2026-09-15T12:00:00Z", +}; + +const READ_ONLY_KEYS = ["id", "entity_id", "creator_id", "created_at", "updated_at"] as const; + +function withoutReadOnlyKeys(stored: TransformTest): Record { + const body: Record = { ...stored }; + for (const key of READ_ONLY_KEYS) { + delete body[key]; + } + return body; +} + +describe("TransformTestCreateInput", () => { + it("names the read-only keys a body cloned from a stored test still carries", () => { + const result = TransformTestCreateInput.safeParse(TransformTest.parse(STORED)); + + expect(result.success).toBe(false); + expect(result.error?.issues).toEqual([ + { + code: "unrecognized_keys", + keys: [...READ_ONLY_KEYS], + path: [], + message: 'Unrecognized keys: "id", "entity_id", "creator_id", "created_at", "updated_at"', + }, + ]); + }); + + it("accepts that same body once those keys are stripped", () => { + const body = withoutReadOnlyKeys(TransformTest.parse(STORED)); + + expect(TransformTestCreateInput.parse(body)).toEqual({ + transform_id: 7, + name: "people summary holds one row", + description: null, + inputs: [ + { table: { schema: "public", name: "people" }, format: "sql", sql: "SELECT 1 AS id" }, + ], + expectations: [{ type: "empty", name: "no null ids", sql: "SELECT * FROM out" }], + }); + }); +}); + +describe("TransformTestUpdateInput", () => { + it("names a read-only key rather than sending it for the server to refuse", () => { + const result = TransformTestUpdateInput.safeParse({ id: 3, name: "renamed" }); + + expect(result.success).toBe(false); + expect(result.error?.issues).toEqual([ + { + code: "unrecognized_keys", + keys: ["id"], + path: [], + message: 'Unrecognized key: "id"', + }, + ]); + }); + + it("patches a single field", () => { + expect(TransformTestUpdateInput.parse({ name: "renamed" })).toEqual({ name: "renamed" }); + }); +}); diff --git a/packages/client/src/domain/transform-test.ts b/packages/client/src/domain/transform-test.ts index d16b4a00..4016300a 100644 --- a/packages/client/src/domain/transform-test.ts +++ b/packages/client/src/domain/transform-test.ts @@ -89,6 +89,11 @@ export const TransformTestCompact = TransformTest.pick({ }).strip(); export type TransformTestCompact = z.infer; +/** + * The server closes this map, so an unrecognized key is a 400 there. Refusing it here names the key + * instead: a body cloned from `get` has to shed `id`, `entity_id`, `creator_id`, `created_at` and + * `updated_at` before it can be sent back. + */ export const TransformTestCreateInput = z .object({ transform_id: z.number().int().positive(), @@ -97,9 +102,10 @@ export const TransformTestCreateInput = z inputs: z.array(TransformTestInput), expectations: z.array(TransformTestExpectation), }) - .loose(); + .strict(); export type TransformTestCreateInput = z.infer; +/** Closed for the same reason as [[TransformTestCreateInput]]; every field is optional. */ export const TransformTestUpdateInput = z .object({ transform_id: z.number().int().positive().optional(), @@ -108,7 +114,7 @@ export const TransformTestUpdateInput = z inputs: z.array(TransformTestInput).optional(), expectations: z.array(TransformTestExpectation).optional(), }) - .loose(); + .strict(); export type TransformTestUpdateInput = z.infer; /** From 934bc6eb120c8a08d6aebd401f048ba54aebaf2a Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Tue, 15 Sep 2026 23:43:24 -0400 Subject: [PATCH 3/8] Show both expectation types in the transform-test example Co-Authored-By: Claude Opus 5 --- packages/cli/skill-data/transform/SKILL.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/cli/skill-data/transform/SKILL.md b/packages/cli/skill-data/transform/SKILL.md index e5fbaa15..cda519c8 100644 --- a/packages/cli/skill-data/transform/SKILL.md +++ b/packages/cli/skill-data/transform/SKILL.md @@ -214,6 +214,11 @@ mb transform-test run --profile --json # exits non-zero u "format": "rows", "columns": [{ "name": "id", "database_type": "INTEGER" }], "rows": [{ "id": 1 }] + }, + { + "type": "empty", + "name": "no minor survived the filter", + "sql": "SELECT * FROM public.adults WHERE age < 18" } ] } From 3fbcf8b58b277dccafad33f2282d2eb0eb85c84e Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Tue, 15 Sep 2026 23:44:58 -0400 Subject: [PATCH 4/8] Give the two example expectations different jobs `equals` pins the exact rows; `empty` asserts an invariant the row data alone would not show. Co-Authored-By: Claude Opus 5 --- packages/cli/skill-data/transform/SKILL.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/skill-data/transform/SKILL.md b/packages/cli/skill-data/transform/SKILL.md index cda519c8..277046c3 100644 --- a/packages/cli/skill-data/transform/SKILL.md +++ b/packages/cli/skill-data/transform/SKILL.md @@ -210,15 +210,15 @@ mb transform-test run --profile --json # exits non-zero u "expectations": [ { "type": "equals", - "name": "keeps only the adult", + "name": "exactly one row, id 1", "format": "rows", "columns": [{ "name": "id", "database_type": "INTEGER" }], "rows": [{ "id": 1 }] }, { "type": "empty", - "name": "no minor survived the filter", - "sql": "SELECT * FROM public.adults WHERE age < 18" + "name": "no null ids", + "sql": "SELECT * FROM public.adults WHERE id IS NULL" } ] } From 930e847efafd2006d1405108f72c4100c6d5b04c Mon Sep 17 00:00:00 2001 From: Timothy Dean Date: Thu, 17 Sep 2026 14:48:44 -0600 Subject: [PATCH 5/8] Rename database_type to cast_type --- packages/cli/skill-data/transform/SKILL.md | 10 ++++++---- packages/client/src/domain/transform-test.ts | 9 ++++++++- packages/client/src/resources/transform-test.test.ts | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/cli/skill-data/transform/SKILL.md b/packages/cli/skill-data/transform/SKILL.md index 277046c3..9f805e15 100644 --- a/packages/cli/skill-data/transform/SKILL.md +++ b/packages/cli/skill-data/transform/SKILL.md @@ -185,7 +185,9 @@ mb transform-test delete --yes --profile mb transform-test run --profile --json # exits non-zero unless it passes ``` -**`inputs`** — one per table the transform reads, each naming a `table` plus either `format: "sql"` with `sql`, or `format: "rows"` with `columns` (each a `name` and a `database_type` the warehouse takes as a `CAST` target) and `rows`. +**`inputs`** — one per table the transform reads, each naming a `table` plus either `format: "sql"` with `sql`, or `format: "rows"` with `columns` (each a `name` and a `cast_type`) and `rows`. + +**`cast_type` is a `CAST` target, not a column type**, and the two vocabularies differ per warehouse: MySQL casts to `SIGNED` and reports `INTEGER`; ClickHouse takes `Nullable(Int32)` for a column that is `Int64`. So a body is warehouse-specific — don't copy a `database_type` out of a run result into a `cast_type`, and don't expect one body to run everywhere. **`expectations`** — `type: "empty"` with the `sql` that must return no rows, or `type: "equals"`, which needs the same `format` split as an input (`"rows"` with `columns`/`rows`, or `"sql"` with a query). An `equals` without a `format` is refused. @@ -198,8 +200,8 @@ mb transform-test run --profile --json # exits non-zero u "table": { "schema": "public", "name": "people" }, "format": "rows", "columns": [ - { "name": "id", "database_type": "INTEGER" }, - { "name": "age", "database_type": "INTEGER" } + { "name": "id", "cast_type": "INTEGER" }, + { "name": "age", "cast_type": "INTEGER" } ], "rows": [ { "id": 1, "age": 30 }, @@ -212,7 +214,7 @@ mb transform-test run --profile --json # exits non-zero u "type": "equals", "name": "exactly one row, id 1", "format": "rows", - "columns": [{ "name": "id", "database_type": "INTEGER" }], + "columns": [{ "name": "id", "cast_type": "INTEGER" }], "rows": [{ "id": 1 }] }, { diff --git a/packages/client/src/domain/transform-test.ts b/packages/client/src/domain/transform-test.ts index 4016300a..3d493681 100644 --- a/packages/client/src/domain/transform-test.ts +++ b/packages/client/src/domain/transform-test.ts @@ -7,10 +7,16 @@ const TransformTestTable = z }) .loose(); +/** + * A column of declared test data. `cast_type` is what the cell is cast to, so it has to satisfy the + * warehouse's CAST grammar rather than name one of its column types — MySQL takes `SIGNED` and + * reports `INTEGER`. Contrast [[TransformTestResultColumn]], whose `database_type` is a real column + * type; the two are not interchangeable. + */ const TransformTestColumn = z .object({ name: z.string().min(1), - database_type: z.string().min(1).describe("The warehouse's own spelling, e.g. VARCHAR(255)"), + cast_type: z.string().min(1).describe("A CAST target, e.g. VARCHAR(255) or MySQL's SIGNED"), }) .loose(); @@ -124,6 +130,7 @@ export type TransformTestUpdateInput = z.infer; export const TransformTestStatus = z.enum(["passed", "failed", "error"]); export type TransformTestStatus = z.infer; +/** A column as a run reports it: the output table's own type, not the declared `cast_type`. */ const TransformTestResultColumn = z.object({ name: z.string(), database_type: z.string() }).loose(); const TransformTestCellMismatch = z diff --git a/packages/client/src/resources/transform-test.test.ts b/packages/client/src/resources/transform-test.test.ts index 7090bdb3..066e760a 100644 --- a/packages/client/src/resources/transform-test.test.ts +++ b/packages/client/src/resources/transform-test.test.ts @@ -25,7 +25,7 @@ const TRANSFORM_TEST = { { table: { schema: "public", name: "people" }, format: "rows", - columns: [{ name: "id", database_type: "INTEGER" }], + columns: [{ name: "id", cast_type: "INTEGER" }], rows: [{ id: 1 }], }, ], From 028fd2899e40772ab2484ac0dc19f85104789fb7 Mon Sep 17 00:00:00 2001 From: Timothy Dean Date: Thu, 17 Sep 2026 14:57:51 -0600 Subject: [PATCH 6/8] Check in an adapted form of DWT skill --- packages/cli/skill-data/core/SKILL.md | 21 +- .../skill-data/transform-test-plan/SKILL.md | 189 +++++++++++++ .../references/checklist.md | 81 ++++++ .../transform-test-plan/references/checks.md | 259 ++++++++++++++++++ 4 files changed, 540 insertions(+), 10 deletions(-) create mode 100644 packages/cli/skill-data/transform-test-plan/SKILL.md create mode 100644 packages/cli/skill-data/transform-test-plan/references/checklist.md create mode 100644 packages/cli/skill-data/transform-test-plan/references/checks.md diff --git a/packages/cli/skill-data/core/SKILL.md b/packages/cli/skill-data/core/SKILL.md index db8fb953..27071b78 100644 --- a/packages/cli/skill-data/core/SKILL.md +++ b/packages/cli/skill-data/core/SKILL.md @@ -6,7 +6,7 @@ allowed-tools: Read, Write, Edit, Bash, AskUserQuestion # metabase-cli (core) -The official Metabase CLI (`mb`) drives a Metabase instance over its REST API: auth, list/get/create/update/delete on every resource, query and transform execution, content search, git-sync (representations ↔ instance), and entity-id translation. +`mb` drives a Metabase instance over its REST API: CRUD on every resource, query and transform execution, search, git-sync (representations ↔ instance), and entity-id translation. Top-level command groups (run `mb --help` to discover verbs): @@ -16,7 +16,7 @@ document | timeline | timeline-event | transform | transform-job | transform-tag search | git-sync | setup | eid | uuid | upgrade | skills ``` -The conventions below — auth, flags, output, body input — hold across **every** group. Per-command flags and examples live in each command's `--help`; add `--json` for the machine-readable form with the output JSON Schema. A few flows have their own skills (see "Specialized skills"). When a card needs a query, prefer MBQL over native SQL (portable, pre-flight-validated — load `mbql`); fall back to native SQL when MBQL can't express it. +The conventions below — auth, flags, output, body input — hold across **every** group. When a card needs a query, prefer MBQL over native SQL (portable, pre-flight-validated — load `mbql`); fall back to native SQL when MBQL can't express it. ## Auth & profiles @@ -28,7 +28,7 @@ mb auth status --json # → {profile, present, url} for the d mb auth status --profile --json # health probe for one profile ``` -`auth list` is the primary enumeration path — one call returns every profile with sanitized URL, an `authenticated` flag, and a probe `status` (`ok` / `auth-failed` / `network-error` / `server-error` / `not-probed`). Use it before asking which profile to pick. +`auth list` is the primary enumeration path — one call returns every profile with sanitized URL, an `authenticated` flag, and a probe `status` (`ok` / `auth-failed` / `network-error` / `server-error` / `not-probed`). - One profile and intent doesn't disambiguate → use it. - Several → ask via `AskUserQuestion`, presenting the names from `auth list`. @@ -60,7 +60,7 @@ Every list/get verb supports the same output flags: - `--json` — emit the full JSON envelope, safe for `jq`. Default is human-readable text. - `--full` — include every field (the compact projection is the default, and is the agent-facing contract). - `--fields a,b.c.d` — project specific dot-paths. Mutually exclusive with `--full`. **Paths are relative to each `data[]` item on list verbs, and to the root on single-item verbs.** So it's `--fields id,name` on `… list` / `database schema-tables` (`data.id` and `data[].id` both fail with `unknown field path: "data.id"`), and `--fields id,name,display` on `card get`, `--fields data.rows` on `mb query` (whose `data` is an object). -- `--max-bytes ` — cap output size. Default 24576 (sized to fit under agent-harness tool-output limits); `0` disables. On a list it drops trailing items and sets `truncated` (see below). Single-item commands (`get`) never truncate — over the cap they throw a `ConfigError` (exit 2: "output is N bytes, over the M-byte --max-bytes cap; …") whose tail names the remedy: on schema-shaped commands it is the exact narrower command to run instead — follow it rather than raising the cap. +- `--max-bytes ` — cap output size. Default 24576; `0` disables. On a list it drops trailing items and sets `truncated` (see below). Single-item commands (`get`) never truncate — over the cap they throw a `ConfigError` (exit 2: "output is N bytes, over the M-byte --max-bytes cap; …") whose tail names the remedy: on schema-shaped commands it is the exact narrower command to run instead — follow it rather than raising the cap. - JSON output is a single line when stdout is piped (pretty-printed only at a TTY) — always parse it, never scrape by line position. ## List windows and resumption @@ -69,10 +69,10 @@ Every list verb takes `--limit ` (items this call returns) and `--offset ` - **`has_more` decides whether to keep going — never compare counts.** `total` is the server's count on endpoints that report one and `null` on those that don't, so arithmetic over it is not a termination condition. - **To continue, pass `next_offset` back as `--offset`.** When `has_more` is true `next_offset` is past the offset you sent, so the loop advances; when false the walk is over and `next_offset` is `null`. -- **`truncated` means the byte cap cut the output, not that the data ran out.** `has_more`/`next_offset` are recomputed to the cut point, so a capped list resumes like any window. Its `bytes` is what the untruncated answer would have measured, so it sizes the work left rather than the reply you hold. Narrow rows with `--fields` rather than raising `--max-bytes` — a bigger cap spends context on fields you didn't ask for, and the cap counts only what you asked for, so `--fields` buys rows directly. A capped list always returns at least one row; when not even one fits it exits 2 with "the smallest response this list can produce is N bytes, over the M-byte --max-bytes cap; …". +- **`truncated` means the byte cap cut the output, not that the data ran out.** `has_more`/`next_offset` are recomputed to the cut point, so a capped list resumes like any window. Its `bytes` is what the untruncated answer would have measured, so it sizes the work left rather than the reply you hold. Narrow rows with `--fields` rather than raising `--max-bytes` — the cap counts only what you asked for, so `--fields` buys rows directly. A capped list always returns at least one row; when not even one fits it exits 2 with "the smallest response this list can produce is N bytes, over the M-byte --max-bytes cap; …". - `limit` is echoed only when you passed `--limit` — except `mb search`, which defaults to `--limit 20` (an unbounded search is expensive server-side) and so always reports one. On nouns the server doesn't page, one large `--limit` with narrow `--fields` is a single request; many small `--offset` hops are one request each. -The whole walk, literally: +The whole walk: ```bash offset=0 @@ -104,7 +104,7 @@ mb create --file ./.scratch/body.json --profile --json Single-quoted `'EOF'` stops the shell interpolating `$vars` inside the JSON. -Write working files to **`./.scratch`** in the current directory (`mkdir -p ./.scratch` first), never `/tmp` — better permissions, they persist across the session, and the user can review them. +Write working files to **`./.scratch`** in the current directory (`mkdir -p ./.scratch` first), never `/tmp` — they persist across the session and the user can review them. ## Discovering commands and schemas @@ -123,7 +123,7 @@ mb transform --help --json | jq -r '.commands[].command' # verbs under "transfo ## Resource quirks worth memorizing -Routine verb shapes (list / get / create / update), every flag, and output schemas live in each command's `--help` (add `--json` for output schemas). Below is only what help does _not_ tell you: footguns and non-obvious behaviors. +Only what `--help` does _not_ tell you: footguns and non-obvious behaviors. - **db traversal: the hydration ladder.** Start with `database get --include tables` — the compact table map (id, name, schema, description per table), one call that fits most databases. Pick the relevant tables, then `table fields ` per table (bounded: fields are per-table). `--include tables.fields` is the full rollup — small databases only. Hundreds of tables? Traverse by schema (`database schemas ` → `database schema-tables `) or look tables up by name (`search --models table --db-id --limit 10`). `sync-schema` / `rescan-values` queue async work and return `{status:"ok"}` immediately; `sync-schema --wait` blocks until `initial_sync_status: complete`. - **table fields.** `table get` never returns fields on its own — pass `--include fields` (compact; the underlying query_metadata response also carries FK targets and dimensions, visible under `--full`) or use `table fields ` (list envelope). `table update` patches table-level metadata only; physical columns aren't editable. @@ -131,7 +131,7 @@ Routine verb shapes (list / get / create / update), every flag, and output schem - **upload (CSV → tables).** `upload csv --file ` creates a new table + model (prints `{model_id, table_id}`); `upload append ` / `upload replace --file ` add to / overwrite a table **previously created by upload** (columns must match). The destination db+schema is admin-configured, not per-call — check with `mb setting get uploads-settings --json` (`db_id: null` ⇒ uploads off/unconfigured; needs admin to read). `--collection ` only sets the model's collection. Max 50 MB. Errors: **"The uploads database is not configured."** = no db has uploads enabled; **"Uploads are not enabled."** = the append/replace target isn't an uploaded table. - **content-translation.** EE-only (`content_translation` premium feature), admin-only, and separate from Remote Sync. `content-translation download > translations.csv` streams the complete active dictionary; `content-translation upload --file translations.csv` replaces every active translation with the file's contents. Always upload the canonical complete CSV, never a partial patch. An empty dictionary downloads as Metabase's four-row sample dictionary — don't re-upload it as real translations. Metabase limits dictionaries to 1.5 MiB. - **card.** `dataset_query` is the **flat** `mbql/query` value, not a legacy `{type:"query",query:…}` envelope (→ `mbql`). `--export-format csv|xlsx` streams the raw export (pipe to a file), bypassing the JSON envelope. `archive` is the only delete; unarchive with `update --body '{"archived":false}'`. `visualization_settings` keys are scoped by `display` and aren't pre-flighted — see `visualization`. -- **dashboard.** Dashcards round-trip through `PUT /api/dashboard/:id` (no per-dashcard endpoint): `update-dashcard ` patches one safely; `update --body '{"dashcards":[…]}'` replaces the whole set (omitted ids are deleted server-side; negative ids for new cards). Every dashcard must include `card_id`, including existing rows; use `card_id:null` plus a `visualization_settings.virtual_card` block (`{display:"text"|"heading"|"link"|…}`) for non-question cards. `create` accepts the **same** `dashcards` array in its initial body, so lay out the whole dashboard in one call. `create`/`update` pre-flight every positive `card_id` and exit **2** with `{ok:false,errors:[…]}` on a bad ref (non-bypassable). `dashboard get ` (or `--full`) hydrates dashcards/tabs; `list` omits them. **The grid is 24 columns wide:** each dashcard's `{col, row, size_x, size_y}` is in grid units — **full-width is `size_x: 24`** (`size_x: 12` is half a row, the usual cause of a card filling only half the width). Keep `col + size_x ≤ 24`, start a full-width stack's `col` at 0, and don't overlap (the server stores collisions as sent — no auto-fix). Layout patterns and per-chart default sizes → the `dashboard` skill; load it before composing any `dashcards` array. +- **dashboard.** Dashcards round-trip through `PUT /api/dashboard/:id` (no per-dashcard endpoint): `update-dashcard ` patches one safely; `update --body '{"dashcards":[…]}'` replaces the whole set (omitted ids are deleted server-side; negative ids for new cards). Every dashcard must include `card_id`, including existing rows; use `card_id:null` plus a `visualization_settings.virtual_card` block (`{display:"text"|"heading"|"link"|…}`) for non-question cards. `create` accepts the **same** `dashcards` array in its initial body, so lay out the whole dashboard in one call. `create`/`update` pre-flight every positive `card_id` and exit **2** with `{ok:false,errors:[…]}` on a bad ref (non-bypassable). `dashboard get ` (or `--full`) hydrates dashcards/tabs; `list` omits them. **The grid is 24 columns wide:** each dashcard's `{col, row, size_x, size_y}` is in grid units — **full-width is `size_x: 24`** (`size_x: 12` is half a row, the usual cause of a card filling only half the width). Keep `col + size_x ≤ 24`, start a full-width stack's `col` at 0, and don't overlap (the server stores collisions as sent — no auto-fix). Layout patterns and per-chart default sizes → the `dashboard` skill. - **dashboard parameters (filters).** A dashboard's `parameters` array holds its filter widgets; they're part of the dashboard record, so read them with `dashboard get --fields parameters --json` (no separate verb). **Editing replaces the _whole_ array** (like dashcards), so it's a read-modify-write loop and omitting a parameter deletes it. A parameter only filters a card once it is **mapped** onto that dashcard's `parameter_mappings` — an unmapped parameter is an inert widget. `type` is a **closed enum**; an unlisted value is a hard parse error that echoes the full allowed set back to you. `dashboard parameter-values [--query ]` fetches a widget's selectable values (`{values, has_more_values}`; `--query` is a case-insensitive substring search). Parameter types, ids, mapping targets, and value sources → the `dashboard` skill; load it before authoring a `parameters` array. - **alert / subscription are two unrelated systems.** `alert` watches one **card** and fires on a send condition (`/api/notification`: a cron string, `channel/email`-prefixed handlers, typed recipients); `subscription` delivers one **dashboard** on a schedule (`/api/pulse`: structured `schedule_type` + hour/day/frame, bare `email` channels, `{id}|{email}` recipients). The bodies are not interchangeable. Both silently deliver nowhere if the server has no SMTP / Slack app — check `mb setting get 'email-configured?'` (quote it; the `?` is a shell glob) before creating either. Their list-valued fields (`handlers`/`subscriptions`, `channels`/`cards`) **replace wholesale** on update, so adding one recipient is a read-modify-write. `mb card alerts ` and `mb dashboard subscriptions ` list what's already attached to a card/dashboard; `archive` deactivates rather than deletes. Load the `notification` skill before authoring either body. - **snippet `--archived` is a swap, not a union** — list returns _either_ active _or_ archived rows, never both. (Same for `--filter archived` on dashboard/collection.) @@ -143,7 +143,7 @@ Routine verb shapes (list / get / create / update), every flag, and output schem - **transform.** Iterate with `transform update `, never `delete` + `create` (keeps the row, `entity_id`, materialized table, and YAML filename — avoids `_2` suffixes and noisy git history). `transform run` needs `--wait` (or `--sync`, which also waits for the output table to register and returns `target_table_id`) or you get only `{run_id, final:null}`. `transform-test run ` checks a transform against fixtures in temp tables — no real table is read — and exits non-zero unless it passes (v64+). (→ `transform`.) - **setup is one-shot.** `mb setup` walks `/api/setup` for a **fresh** instance only — errors against an already-configured one. Mostly for bootstrapping local / e2e instances. - **eid** translates a string entity id → numeric id: `mb eid --model --json`. Entity ids are NanoIDs that can start with `-`, which the positional form misreads as a flag (shell quotes don't help) — for those, use `--body '{"entity_ids":{"card":["-…"]}}'` (the id is a JSON string value, immune to flag parsing). -- **library.** EE-only (`library` premium feature, v59+). The Library is a curated subtree (`library-data` "Data" + `library-metrics` "Metrics" under a `library` root): tables published to **Data** appear first in data pickers and rank up in search; metrics saved to **Metrics** are prioritized in nav, search, and the query builder — it's how you tell people (and agents) "start from these, they're trusted." `library get` shows the Library and its Data/Metrics collection ids; `library create` provisions it (idempotent). `library publish --table-ids/--db-ids/--schemas` publishes tables into Data — it **resolves the Data collection itself and creates the Library if absent** (no collection id to find); each `--schemas` entry is `:` (e.g. `1:public`), not a bare name. `publish` cascades to upstream FK dependencies, `unpublish` to downstream dependents; both need **admin or data-analyst** (Curate alone won't publish) and exit **403** without write **and** query permission on every affected table. Publish status shows on the table: `table get`/`table list` carry `is_published` (`collection_id` under `--full`). Good candidates are finished, analysis-ready tables — clean/combine via transforms first, then publish the polished result. Publishing does not put the Data collection in the git-sync scope: on a remote-sync instance, `mb git-sync add-collection ` is what makes exports carry the published tables' metadata (see the `git-sync` skill). +- **library.** EE-only (`library` premium feature, v59+). The Library is a curated subtree (`library-data` "Data" + `library-metrics` "Metrics" under a `library` root): tables published to **Data** appear first in data pickers and rank up in search; metrics saved to **Metrics** are prioritized in nav, search, and the query builder. `library get` shows the Library and its Data/Metrics collection ids; `library create` provisions it (idempotent). `library publish --table-ids/--db-ids/--schemas` publishes tables into Data — it **resolves the Data collection itself and creates the Library if absent** (no collection id to find); each `--schemas` entry is `:` (e.g. `1:public`), not a bare name. `publish` cascades to upstream FK dependencies, `unpublish` to downstream dependents; both need **admin or data-analyst** (Curate alone won't publish) and exit **403** without write **and** query permission on every affected table. Publish status shows on the table: `table get`/`table list` carry `is_published` (`collection_id` under `--full`). Publish finished, analysis-ready tables — clean/combine via transforms first. Publishing does not put the Data collection in the git-sync scope: on a remote-sync instance, `mb git-sync add-collection ` is what makes exports carry the published tables' metadata (see the `git-sync` skill). - **query / uuid.** `mb query` is the ad-hoc MBQL surface (`--print-schema` → `--dry-run` → run); `mb uuid --count ` mints the `lib/uuid` values MBQL clauses need. Both live in `mbql`. ## Specialized skills (load on demand) @@ -157,6 +157,7 @@ This file is enough for any single-command task. For anything deeper, load the r - **`metadata`** — setting field/table metadata: semantic types, foreign-key targets, dropdown/scan behavior, and column visibility, and the downstream features each unlocks. Load when editing what a column _means_, not its data. - **`notification`** — scheduled delivery: question alerts (`mb alert`) and dashboard subscriptions (`mb subscription`). Choosing between them, the two schedule/recipient contracts, channel prerequisites, testing a send. - **`transform`** — transform body JSON, create + run-with-wait, run inspection, tags, jobs. +- **`transform-test-plan`** — deciding _what_ to test in a transform: the fixture cast, the expectations, the coverage matrix. (`transform` has the `mb transform-test` shapes.) - **`document`** — Metabase documents (TipTap body, embedding cards). - **`git-sync`** — round-tripping content to/from a git remote. - **`data-workflow`** — the guided, end-to-end data workflow: investigate raw data, build clean analysis-ready tables, define reusable segments/measures/metrics, answer questions, build dashboards. **Start here when the user states a goal rather than a single verb** — "make sense of my data", "build a data model", "go from raw data to a dashboard", "be my data analyst", "set up analytics for X". It detects where the data is and routes to the right stage. diff --git a/packages/cli/skill-data/transform-test-plan/SKILL.md b/packages/cli/skill-data/transform-test-plan/SKILL.md new file mode 100644 index 00000000..d404e7cd --- /dev/null +++ b/packages/cli/skill-data/transform-test-plan/SKILL.md @@ -0,0 +1,189 @@ +--- +name: transform-test-plan +description: Derive a comprehensive test plan for a transform — the fixture cast, the expectations, hand-derived expected rows, and a coverage matrix — from the model's declared design. Covers input partitioning (zero-case, multiplicity, dirty rows), grain / conservation / recomputation / conformance checks, and known-quirk conventions. Load when the user wants tests planned or written for transforms — "write tests for my transforms", "is my model right", "test plan for this pipeline", "add data quality checks" — whether the model is mid-build or already deployed. The `mb transform-test` command and body shapes live in the `transform` skill; this one decides what to test. +allowed-tools: Read, Write, Edit, Bash, AskUserQuestion +--- + +# Planning transform tests + +Turn a transform into a fixture cast that proves the logic on small known rows, expectations that +state the model's invariants, and a coverage matrix showing what's checked and what's deliberately +not. Mechanics — the `inputs`/`expectations` body shape and every `mb transform-test` verb — live in +the `transform` skill (`mb skills get transform`); load it before authoring, and never restate it +here. + +Every check derives from what the model **declares** — detected from its SQL, confirmed with its +owner — never from conformance to a modeling doctrine. One plan serves two moments: while the model +is **built**, checks pin each design decision; once **deployed**, the same SQL screens production +tables for anomalies. + +## Operating rules + +- **Detect, then derive.** Classify what the transform is and which conventions it uses (the + checklist); derive checks only from that. Star, one-big-table, partial denormalization — all + fine; never flag a style. +- **Judgment calls go through the checklist.** The session's autonomy setting governs which answers + you supply yourself and which you bring to the user — it never makes the checklist a formality. + Every answer you supply yourself is recorded in the plan as a stated assumption, paired with the + named expectation that enforces it — reversing the decision then breaks a test, not a paragraph. + And regardless of setting, when genuinely unsure, ask — a wrong-but-confident grain poisons every + downstream check. +- **Expected rows are derived by hand** from the fixture story and business meaning — never captured + from the transform's output, which asserts only that the transform equals itself. +- **Batteries stay off until declared structure switches them on.** No snapshot-density checks + without a snapshot, no version-history checks without effective/end/current columns. An empty + section beats a speculative one. + +## The procedure + +1. **Profile the real inputs**: per table, row count; per column, min/max/distinct-count/null + incidence; orphan counts across declared links (`mb field summary`, `mb query`). Profiling feeds + domains, bounds, null partitions, and key candidates — and every fixture edge cites the real-data + condition that warrants it, with its count ("the warehouse has 67 ship-before-order rows"). +2. **Detect.** Read the transform's SQL (`mb transform get --full --json`) for: grain + candidates (`GROUP BY` keys, the joins' driving table), join types (orphan handling), correlated + aggregates (stored aggregates), `_` naming (copies), effective/end/current columns + (version history), `now()`/`current_date` (volatile columns). A transform whose grain you cannot + state in one phrase is itself a finding — raise it before writing any test. +3. **Confirm.** Walk [references/checklist.md](references/checklist.md). Three stages: classify the + model, per-table declarations, per-column declarations. Each question carries its detection hint; + answer autonomously where the hint resolves, ask where it doesn't. In build-along mode these are + design questions — treat an undecided answer as a decision to make together, not a blocker. +4. **Derive.** Route every output column through [references/checks.md](references/checks.md) — + declared property → expectation shape, fixture implication, expected-row convention. Read it in + full once per plan; it is the plan's content. +5. **Design the fixture cast**: one small cast per transform (≈5–10 rows per table), human-named + rows ("Alice Premium"), every row a named edge — zero-case entities for every outer join and + aggregation, ≥2-member groups for every grouping and join, one dirty row per screenable defect, + boundary dates. Document it as a table (row → attributes → purpose) in the plan. + + Write each input as `format: "rows"` — the columns and their values _are_ the cast, and the story + stays readable in the body. Reach for `format: "sql"` only when the cast is genuinely easier as a + query — a generated series, say. + + Every column carries a `cast_type`: the target its cells are cast to, which answers to the + warehouse's `CAST` grammar rather than naming one of its column types. MySQL casts to `SIGNED` + and reports the column as `INTEGER`; ClickHouse takes `Nullable(Int32)` where the column is + `Int64`. So a cast type is a per-warehouse choice — a cast written for one engine does not carry + to another, and the `database_type` a run reports is the output column's real type, never a + `cast_type` to paste back. + +6. **Hand-derive the expected rows**, arithmetic recorded in the plan (premium: 3 orders / 350.50 / + 3.0). Every fixture row's fate appears in some expected cell. Pin NULL-vs-0-vs-empty for every + zero-case row — that cell is the null policy's only enforcement. +7. **Author the test.** One test per transform, per coherent story: an `equals` pinning the output, + and `empty` expectations stating the invariants that survive a change to the cast. + + ```bash + mb transform-test create --file ./.scratch/orders-test.json --profile --json + mb transform-test run --profile --json + ``` + + Iterate with `mb transform-test update --file`, never delete + create — the same rule the + `transform` skill states for transforms, and for the same reasons: the row, its `entity_id` and + its YAML filename all survive. + + Tests live with the transform: they serialize and git-sync with it. Give each expectation a name that states the invariant, because the name is what a + failure leads with ("revenue never negative", not "check 3"), and open each `empty` expectation's + SQL with a `--` contract comment: the invariant, and the failure modes it catches ("catches both + dropped orders and join fan-out"). + + Where one invariant applies to several transforms, duplicate the SQL. Nothing is shared between + tests, so give each copy its own name and comment rather than one that only makes sense next to + its twin. + +8. **Emit the coverage matrix** in the plan: rows = output columns with the table's grain; columns = + check classes (grain, conservation, recomputation, conformance, domains, referential integrity, + temporal, screens); cells name the covering expectation or expected-row cell, or state + `gap: `. Scan shared-attribute columns across transforms for cross-table agreement + obligations. Empty cells are honest; silent gaps are not. +9. **Prove the tests have teeth.** Once per test: corrupt one expected cell, `run`, confirm + `cell-mismatches` names exactly that column; revert. Then perturb one input cell and confirm + exactly the declared output cells move. Every `empty` expectation passes against an empty output, + so a green test can still be vacuous. + +## Severity: error, or the tolerated oddity + +Every expectation is pass or fail, and one failure fails the run — `run` exits non-zero, so a test +is a CI gate. **error** = forbidden, and it becomes an `empty` expectation. There is no warn +severity, so **never author an expectation you expect to fire**: a permanently red test trains +everyone to ignore the result and holds the gate down for everyone else. + +A tolerated-but-surfaced oddity — one the owner lives with, like orphan rows or ship-before-order +dates — gets encoded the two ways that hold: **pin it in the `equals` rows** (an orphan passing +through with NULLs is a cell in the expected output, so reversing the tolerance breaks the test), +and **record it in the plan's known-quirks list** with its real-data count, so the tolerance stays a +conscious choice. + +Under budget pressure cut business-rule checks first, then cross-table structure checks; never +single-column screens (domains, ranges, nulls) — cheapest, and the last line. + +## When a check exposes a live bug + +Non-negotiable: **never soften the test to green** — expected values state correct behavior; matching +them to buggy output documents the bug as intended — and **surface the finding with its blast radius +at both scales**, fixture ("1530.24 of 1600.74 fixture dollars survive") and warehouse ("908 of +2,050 orders dropped"). What happens next follows the session's terms, not a fixed protocol: propose +and apply the fix now (when the user wants it or the autonomy setting covers it), or — when the fix +must wait — hold the correct expectation and record the red in the plan with the minimal fix body, +ready for `mb transform update --file`. A stored test has no red-by-design state, so a deferred +fix must be visible in the plan or the test reads as broken. Either way, once green the test stays +as the regression guard. + +## When the doctrine doesn't apply + +The vocabulary follows Kimball's dimensional modeling (grain, additivity, conformed attributes, +slowly changing dimensions) — precise, widely understood terms. Real models are Kimball-inspired at +most; no check may score adherence: + +- Full calendar date dimensions are rare. Never demand one; test date _semantics_ — ranges, + orderings, volatile derivations. +- Surrogate keys are doctrine, natural-key joins are practice. Test whichever key the model + declares; never flag natural-key joins. +- "No NULL FKs / no NULL attributes" is doctrine routinely dropped. Null policy is three independent + declarations — measures, attributes, FKs — each detected and confirmed, never presumed. +- One-big-table is legitimate: it still has a grain, its copies still need agreement checks, its + functional dependencies still hold. +- A transform-level `ORDER BY` has no testable effect — output tables carry no row order and the + `equals` comparison is a multiset. Flag it as probable dead weight (clustering hints aside); never + write an ordering expectation. + +## One transform at a time + +Each test covers one transform. When the transform under test reads another transform's output, +that target table is an input like any other: declare it and fake it. Derive those rows from the +base transform's own expected output, so the two tests tell one story, and note the coupling in both +plans — changing the base's expected rows means changing this test's input. + +## Worked example, condensed + +`orders` + `customers` → _enriched_orders_ (order grain; LEFT JOIN attaches `customer_name`, +`tier`). + +Cast, 7 orders: two tiers; Alice and Carol with 2 orders each (multiplicity); two never shipped +(zero-case for the shipping join); order 106 shipped before ordered (real oddity, count cited); +order 107's customer_id matches no customer (orphan). Two `rows` inputs, one per source table. + +Derived, per catalog: grain uniqueness on `order_id`; row and amount conservation from input to +output, as scalar subqueries over the target and the seeded input; `tier` domain ⊆ {standard, +premium}; orphan = keep-with-NULLs → an expected row pinning the NULL pass-through; ship-before-order +tolerated → the plan's known-quirks list with the warehouse count. The whole output pinned in one +`equals`, hand-computed, arithmetic in the plan. The inner-vs-LEFT-join bug this cast catches — +unshipped orders silently dropped from revenue — is what the zero-case rows exist for. + +## Don't + +- Don't author expectations before loading the `transform` skill — the body shape, the closed + create/update contract, and the verb flags live there. +- Don't capture expected rows from the transform's own output — hand-derive them or they assert + nothing. +- Don't emit checks for structure the model doesn't declare (snapshot density, version history, + bridge weights) — an inapplicable battery buries real findings. +- Don't let a fixture cast go all-clean — no zero-case, no orphan, no dirty row proves the happy + path and nothing else; the bugs live in the edges. +- Don't write an expectation you expect to fail. +- Don't name a table in an `empty` expectation that is neither the transform's target nor a declared + input. Only those two are rewritten to temp tables; anything else is left exactly as written and + reads the real table. +- Don't surface bare check-ids ("per C3…") to the user — name the check in plain words; the ids are + for your cross-referencing, not their reading. diff --git a/packages/cli/skill-data/transform-test-plan/references/checklist.md b/packages/cli/skill-data/transform-test-plan/references/checklist.md new file mode 100644 index 00000000..58b1b374 --- /dev/null +++ b/packages/cli/skill-data/transform-test-plan/references/checklist.md @@ -0,0 +1,81 @@ +# Confirmation checklist + +The judgment calls a plan depends on, in derivation order. Each question carries a **detect** hint — +how to answer it from the SQL, the metadata, or profiling. Answer autonomously where the hint +resolves cleanly; ask where it doesn't. Every answer you supply yourself goes into the emitted plan +as a stated assumption, phrased so the owner can falsify it at a glance ("Assuming `discount` can +never be negative — correct?"). + +In build-along mode these are design questions: the owner is deciding, not recalling. An undecided +answer is a decision to make together — surface the options and the check each implies (the +`checks.md` entry named in parentheses). + +## Stage 1 — classify the model (per transform) + +1. **Grain: one row per what?** (G1, G2) + Detect: `GROUP BY` keys; otherwise the driving table of the joins. Confirm whenever detection and + any written description disagree — that disagreement is itself a finding. +2. **Fact-table type — and if periodic snapshot, dense or sparse?** (F2) + Detect: event-grain with a single event date → transaction; entity×period grain → periodic + snapshot (density is the one yes/no that flips its whole invariant set — always confirm); one row + per pipeline occurrence with milestone-date columns → accumulating snapshot. + +## Stage 2 — per-table declarations + +3. **Conservation ties: output rows/sums tie to which input, with which declared exclusions?** (A2) + Detect: `WHERE` clauses and join types name the exclusions (filtered rows, dropped duplicates). + Exclusions must be declared, or conservation appears to fail. +4. **Zero-group policy: does a group with no contributing rows appear (with zeros) or stay absent?** + (A5) + Detect: driving table of the rollup — grouping the detail can't produce empty groups; joining a + dimension first can. +5. **Orphan handling per join: drop / keep-with-NULLs / default row — and is an orphan tolerated or + forbidden?** (C3, I5) + Detect: join type. INNER = drop, LEFT = keep-with-NULLs, COALESCE to a sentinel = default row. + Tolerated-vs-forbidden is the owner's call — profiling says whether orphans exist today, not + whether they're acceptable. Forbidden becomes an `empty` expectation; tolerated is pinned in the + expected rows and recorded in the known-quirks list. +6. **Version history: effective/end/current columns anywhere? If not, confirm overwrite-everywhere.** + (T1) + Detect: column names. Absence means history is silently rewritten in rollups — state that + consequence when confirming, not just the mechanism. +7. **Does this transform read another transform's output?** (A5) + Detect: another transform's target table in this one's SQL. It becomes a declared input like any + other, faked from that transform's own expected output — confirm which rows, and record the + coupling in both plans. + +## Stage 3 — per-column declarations + +8. **Each measure: additive, semi-additive, or non-additive?** (A1) + Detect: sums and counts are additive; balances and levels are semi-additive; ratios, rates, and + unit prices are non-additive. Confirm the ambiguous ones (a "score"? a "quantity on hand"?). +9. **Legal bounds per measure?** (Q1) + Detect: profiling min/max suggests, business meaning decides — can `discount` be negative? can + `quantity` be zero? Derived bounds are free (a sum of non-negatives is non-negative). +10. **Each derived measure: the exact recomputation rule?** (A3) + Detect: the SELECT expression is the rule — but rounding, business-day adjustments, and NULL + handling are conventions to confirm, not read. +11. **Which columns are stored aggregates, over which detail and filter?** (A4) + Detect: correlated subqueries / joined-aggregate CTEs in the SQL. +12. **Which categorical columns have closed domains — enumerate; is NULL a member?** (C2) + Detect: profiling distinct values gives today's set; the owner confirms it's closed rather than + merely small so far. +13. **Which columns are denormalized copies, of which owning attribute?** (C1) + Detect: `_` naming; any column functionally dependent on a non-grain key. Every hit + must be classified: copy (C1), stored aggregate (A4), or smuggled coarser-grain fact — a + finding. +14. **Which many-to-one edges hold within one output table?** (C4) + Detect: hierarchy-shaped column pairs (product/category, zip/state). Applies where no owning + table is in scope; otherwise C1 covers it. +15. **Which date orderings are business-guaranteed vs. known-violated and tolerated (with real-data + counts)?** (T2) + Detect: profiling counts the violations that exist; the owner decides tolerated vs. bug. +16. **Null policy per column — measures, attributes, FKs separately.** (Q2) + Detect: the SQL shows what's produced (COALESCE, CASE); the owner confirms intent — "count of + nothing": 0 or NULL? empty date: NULL or sentinel? +17. **Which columns are volatile across runs?** (T3) + Detect: `now()` / `current_date` / run-metadata expressions in the SQL. A volatile column is left + out of the `equals` columns; its form can still be asserted with an `empty`. +18. **Per tolerated oddity: does it stay tolerated, and what is its current count?** (Q3) + Detect: can't — this is the conscious-choice question, and the plan's known-quirks list is where + it survives. diff --git a/packages/cli/skill-data/transform-test-plan/references/checks.md b/packages/cli/skill-data/transform-test-plan/references/checks.md new file mode 100644 index 00000000..f046cf5a --- /dev/null +++ b/packages/cli/skill-data/transform-test-plan/references/checks.md @@ -0,0 +1,259 @@ +# Check catalog + +Every check the plan can derive, grouped by theme. Per entry: **when it applies** (the declared +property that switches it on — blank means always), **assert** (the expectation shape), and +**fixtures** (what the input cast must contain for the check to have teeth). + +Expectation SQL names the transform's **target table** and its **declared input tables** under their +real names; both are rewritten to the run's temp tables, so per-row recomputation joins against +inputs are writable. Anything else you name is left exactly as written and reads the real table. + +Ids (G1, A2, …) are for cross-referencing within the plan documents only — never surface them to the +user bare. + +Throughout, `` stands for the transform's target table as written in the transform's own +definition, and `` for a declared input table. + +## Grain & keys + +**G1 — Declared grain.** Every output table states "one row per X"; the declaration anchors every +other check. + +- Applies: always. Elicit or detect (GROUP BY keys; the join's driving table). +- Assert: nothing directly — G1 is the plan's opening move. A table with no statable grain, or a + grain stated inconsistently between docs and SQL, is a finding before any SQL runs. +- Fixtures: the grain declaration decides the cast's row structure. + +**G2 — Grain-key uniqueness.** The grain key stays a key; doubles as the fan-out guard for every +enriching join. + +- Applies: always — one-big-table, rollups, and stars all have a grain. +- Assert (`empty`): `SELECT , COUNT(*) FROM GROUP BY HAVING +COUNT(*) > 1`. NULL grain values form their own bucket. +- Fixtures: a parent with ≥2 children on every join (I4) is what makes this check able to fail. + +## Additivity & reconciliation + +**A1 — Measure additivity classification.** Additive / semi-additive (balances) / non-additive +(ratios, rates, unit prices) decides which aggregations are valid tests. + +- Applies: every numeric measure; per-column declaration. +- Assert: routes the measure — additive → A2; semi-additive → sum across non-time slices only (a + test summing a balance over time is a bug in the plan); non-additive → A3 recomputation, never + reconciled by summing. +- Fixtures: none directly; one coverage-matrix axis (measure × valid aggregation set). + +**A2 — Conservation reconciliation.** Row counts and additive-measure sums tie from input to output; +catches dropped rows and join double-counting at once. + +- Applies: per declared tie (which input, which declared exclusions). +- Assert (`empty`): independent scalar subqueries compared with `IS DISTINCT FROM`, returning the + mismatched pair: + `SELECT (SELECT SUM(t.m) FROM t) AS output_sum, (SELECT SUM(s.m) FROM s) AS +input_sum WHERE (…) IS DISTINCT FROM (…)`. + Never reconcile via a fact-to-fact join — cardinality is uncontrollable and wrong results are + silent. +- Fixtures: amounts chosen so partial survival is visible (distinct values, odd cents). + +**A3 — Derived-measure recomputation.** Averages, lags, ratios, rounded presentations recompute from +their components per row. + +- Applies: every derived column, with its declared rule (rounding, business-day adjustment, NULL + handling). +- Assert (`empty`): `SELECT FROM t WHERE t. IS DISTINCT FROM `. +- Fixtures: component values whose derivation is non-trivial (a NULL in the AVG, a negative lag). + +**A4 — Stored aggregates on entity tables.** Lifetime/rollup stats carried on an entity-grain table +(`lifetime_orders`, `review_count`, `first_order_date`) equal recomputation from detail — per row +_and_ in total. + +- Applies: columns detected as correlated aggregates over a detail table. +- Assert (`empty`), two per column: per-row — `WHERE t. IS DISTINCT FROM (SELECT +COUNT(*)/SUM(…)/MIN(…) FROM d WHERE d. = t.)`; total — the A2 scalar-pair shape. + Totals alone cancel offsetting per-row errors; the per-row form is the one that catches them. +- Fixtures: an entity with several detail rows and an entity with none (I3/I4). + +**A5 — Rollup consistency.** A layered rollup always agrees with its base. + +- Applies: any transform reading another transform's output. The base is a declared input like any + other, faked with rows taken from the base transform's own expected output. +- Assert (`empty`): every rollup measure equals the corresponding aggregation over the base (A2 + shape); group-set equality both directions — `SELECT FROM EXCEPT SELECT DISTINCT + FROM ` and the reverse, filtered by the declared zero-group policy. +- Fixtures: a group with no contributing rows pins the zero-group policy. + +## Fact-table type + +**F2 — Fact-table type bundle.** Transaction / periodic snapshot / accumulating snapshot each carry +a distinct invariant set. + +- Applies: per fact-shaped output, always classified; **transaction** needs nothing beyond G2 + A2 + (sparsity is legitimate). +- Periodic snapshot, if declared _dense_: `COUNT(*) = |entities| × |periods|` (or the declared + subset); per-entity gap detection in the period series; inactive-period representation (zero vs + NULL) pinned in the expected rows. If sparse, density checks off — the one yes/no flips the whole + set. +- Accumulating snapshot: milestone dates monotone in pipeline order where set (`WHERE < +` per adjacent pair); unset-milestone default (NULL vs sentinel) pinned in the expected + rows; completion flags ∈ {0,1} and consistent with their date's set-ness; lags via A3. +- Fixtures (accumulating): occurrences at every completion stage — none, some, all milestones. + +## Conformance & domains + +**C1 — Denormalized-copy agreement.** Every copied attribute agrees with the owning table's value +for that key. + +- Applies: columns declared as copies (detect: `_` naming; any column functionally + dependent on a non-grain key). Each such column must be a declared copy (this check), a stored + aggregate (A4), or it's a smuggled coarser-grain fact — a finding: it double-counts under + summation. +- Assert (`empty`): `SELECT t., t., d. FROM t JOIN d ON +t. = d. WHERE t. IS DISTINCT FROM d.`. +- Scope: in a test, the copies are produced by the very join under test, so this join-form is near- + tautological — the C4 functional-dependency form plus expected-row cell pinning carries the test. + The join-form against an independently materialized owning table is the _drift_ check, which + belongs to whatever screens the deployed tables. +- Fixtures: copies with distinct values per entity so a crossed join is visible. + +**C2 — Closed-domain screen.** A categorical column's values stay inside the declared enumeration. + +- Applies: per column declared closed (detect from profiling; confirm the set and whether NULL is a + member). +- Assert (`empty`): `SELECT , FROM WHERE IS NOT NULL AND NOT IN +()`. +- Fixtures: every domain value represented where practical; one out-of-domain input row if the + source can produce one (I5). + +**C3 — Referential integrity / orphan policy.** Every FK resolves, or the declared orphan handling is +pinned. + +- Applies: per join, conditioned on the declared response — drop / keep-with-NULLs / default row; + tolerated or forbidden. +- Assert, forbidden (`empty`): `SELECT t. FROM t LEFT JOIN d ON t. = +d. WHERE t. IS NOT NULL AND d. IS NULL`. + Tolerated: not an expectation — pin the orphan's pass-through as a row in the `equals` (NULL in + the copied attributes), and record the tolerance with its warehouse count in the plan. + Default-row convention adds: distinct unknown keys must not collapse into one output row. +- Fixtures: one orphan row (I5). Without it the join direction is untested — an inner join silently + dropping unmatched rows is the classic bug this catches. + +**C4 — Many-to-one consistency.** Each declared many-to-one edge holds within the output (product → +one category; zip → one state). + +- Applies: per declared edge; where an owning table is in scope, C1 subsumes it — this is the + one-big-table variant with nothing to join against. +- Assert (`empty`): `SELECT t., COUNT(DISTINCT t.) FROM t GROUP BY t. +HAVING COUNT(DISTINCT t.) > 1`. +- Fixtures: a violating input row if the source can produce one (I5), pinning the transform's + behavior on dirty input. + +## Temporal & version history + +**T1 — Change handling per attribute.** How the model treats a changed source attribute decides the +temporal fixtures. + +- Applies: detect effective/end/current housekeeping columns. +- Present → version-history battery (`empty` each): per durable key exactly one current row; + `effective < end` per row; intervals contiguous and non-overlapping; current row's end = the + declared far-future default; fact rows join the version whose interval contains the fact date. +- Absent → confirm overwrite-everywhere as a stated assumption (history is silently rewritten in + rollups), and derive the propagation probe: change an attribute in an input, run, assert the + output regrouped. +- Fixtures: a before/after change pair; for version history, an entity with ≥2 versions and a fact + row dated inside each interval. + +**T2 — Date-pair ordering.** Business-guaranteed orderings asserted; known violations recorded. + +- Applies: per declared date pair (ordered ≤ shipped ≤ delivered; signup ≤ first order), each + classified guaranteed vs. tolerated-violated. +- Assert: guaranteed (`empty`) — `SELECT , , FROM WHERE < +`. Tolerated — pin the violating row's downstream arithmetic (a negative lag inside an + average) in the expected rows, and record the real-data count in the plan. +- Fixtures: one violating row for every tolerated ordering. + +**T3 — Volatile columns.** Values that change across runs can't be pinned. + +- Applies: columns derived from `now()`/`current_date` (age), run metadata (load timestamps, batch + ids). +- Assert: leave the column out of the `equals` `columns` list — an undeclared column never enters the + comparison. Separately assert its form where warranted (`empty`): `WHERE age NOT BETWEEN 0 AND +120`. The stable source column (`birth_date`) stays exact in the expected rows. +- Fixtures: none special; the split is the point — omit the volatile, pin the stable. + +## Screens & severity + +**Q1 — Range / sign screens.** Each measure's declared bounds hold. + +- Applies: per bounded measure; bounds from business meaning plus profiling (derived bounds are + free: a sum of non-negatives is non-negative). +- Assert (`empty`): `SELECT , FROM WHERE < OR > ` (one-sided + where only one bound exists). +- Fixtures: boundary values where the bound is business-set. + +**Q2 — Null policy, three ways.** Measures, descriptive attributes, and FKs carry independent null +policies; never presume one from another. + +- Applies: per column, asked separately — "count of nothing": 0 or NULL? empty date: NULL or + sentinel? FK: see C3. +- Assert: declared non-null columns get `WHERE IS NULL` (`empty`). Otherwise the policy is + enforced by the expected-row cell of a zero-case fixture row — NULL vs 0 vs empty is invisible + until a fixture forces the choice into a cell. +- Fixtures: the zero-case row (I3) is the enforcement mechanism. + +**Q3 — Naming & the tolerated oddity.** Every expectation declares its meaning. + +- Applies: always, every expectation. +- Convention: the expectation's `name` states the invariant in plain words, because a failure leads + with it. Each `empty` SQL opens with 1–3 comment lines naming the invariant and the failure modes + it catches. Budget cuts drop business-rule checks first, then cross-table structure checks, never + single-column screens. +- A forbidden condition is an `empty` expectation. A tolerated one is pinned in the expected rows + and recorded in the plan's known-quirks list with its warehouse count. Never author an expectation + designed to fire. + +## Input modeling (fixture-design rules) + +**I1 — Profiling-derived partitions.** Fixture edges and tolerated non-ties cite profiled reality +with counts; the plan's known-quirks section carries each tolerated oddity with its reason and +count. + +**I2 — Shared fixture cast.** One small human-named cast per transform, every row a named edge, +documented as a story table (row → attributes → purpose) in the plan. Where one transform's output +is another's input, derive the faked rows from the first's expected output so the two tests tell one +story. + +**I3 — Zero-case partitions.** For every outer join and aggregation relation: one entity with zero +matches. Its expected row pins the absence representation (Q2) and the join direction (C3) — the +all-clean cast is the single most common cause of vacuous tests. When the schema distinguishes +states the profiled data never exhibits (a literal zero in a source that only has NULLs and +positives), fixture the missing state — nothing else forces it onto an expected cell. + +**I4 — Multiplicity partitions.** Every aggregation gets a >1-member group and an exactly-1 group +(the 0 case is I3); every join gets a parent with ≥2 children. Expected values then differ from any +single row's, so copy-through bugs can't pass. + +**I5 — Dirty-input pinning.** Per screenable defect the source can carry (orphan FK, out-of-domain +value, duplicate natural key, hierarchy violation, ordering violation): one fixture row exhibiting +it, an expected row pinning the transform's response (drop / pass-through / default), and — where +the defect is forbidden — the matching `empty` expectation. Which duplicate survives a dedupe is +invisible until a conflicting-duplicate fixture pins it. + +**I6 — Mutation probes.** Run at plan-validation time, not on every run: corrupt one expected cell → +`cell-mismatches` must name that exact column; perturb one input cell → exactly the declared output +cells move. Both guard against vacuous tests: every `empty` expectation passes against an empty +output. + +## Plan-level artifacts + +**P1 — Coverage matrix.** Output columns (with the table's grain) × check classes; cells name the +covering expectation or expected-row cell, or state `gap: `; shared-attribute columns scanned +across transforms for agreement obligations (C1). + +**P2 — Hand-derived expected rows.** From the fixture story and business meaning, arithmetic recorded +in the plan; never captured from output; every fixture row's fate appears in some expected cell. + +**P3 — Live-bug handling.** A check failing against a deployed transform is a real finding: never +soften the test to green; quantify the damage at fixture and warehouse scale. Then fix now, or record +the red in the plan with the minimal fix body, per the session's terms. A stored test has no +red-by-design state, so a deferred fix must be visible in the plan or the test reads as broken. From a212a8aa235322500fc5b6a861cc5abbbfe64ea8 Mon Sep 17 00:00:00 2001 From: Timothy Dean Date: Thu, 17 Sep 2026 15:45:39 -0600 Subject: [PATCH 7/8] Let the transform skill handle the impl details, keep the test skill to test philosophy and design --- .../skill-data/transform-test-plan/SKILL.md | 39 +++++-------------- packages/cli/skill-data/transform/SKILL.md | 2 + 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/packages/cli/skill-data/transform-test-plan/SKILL.md b/packages/cli/skill-data/transform-test-plan/SKILL.md index d404e7cd..16f02347 100644 --- a/packages/cli/skill-data/transform-test-plan/SKILL.md +++ b/packages/cli/skill-data/transform-test-plan/SKILL.md @@ -57,16 +57,9 @@ tables for anomalies. aggregation, ≥2-member groups for every grouping and join, one dirty row per screenable defect, boundary dates. Document it as a table (row → attributes → purpose) in the plan. - Write each input as `format: "rows"` — the columns and their values _are_ the cast, and the story - stays readable in the body. Reach for `format: "sql"` only when the cast is genuinely easier as a - query — a generated series, say. - - Every column carries a `cast_type`: the target its cells are cast to, which answers to the - warehouse's `CAST` grammar rather than naming one of its column types. MySQL casts to `SIGNED` - and reports the column as `INTEGER`; ClickHouse takes `Nullable(Int32)` where the column is - `Int64`. So a cast type is a per-warehouse choice — a cast written for one engine does not carry - to another, and the `database_type` a run reports is the output column's real type, never a - `cast_type` to paste back. + Express the cast as literal rows rather than as a query, so the story stays legible in the test + itself; a query is worth it only when the rows are mechanical to generate. A cast is written + against one warehouse and does not carry to another. 6. **Hand-derive the expected rows**, arithmetic recorded in the plan (premium: 3 orders / 350.50 / 3.0). Every fixture row's fate appears in some expected cell. Pin NULL-vs-0-vs-empty for every @@ -74,16 +67,7 @@ tables for anomalies. 7. **Author the test.** One test per transform, per coherent story: an `equals` pinning the output, and `empty` expectations stating the invariants that survive a change to the cast. - ```bash - mb transform-test create --file ./.scratch/orders-test.json --profile --json - mb transform-test run --profile --json - ``` - - Iterate with `mb transform-test update --file`, never delete + create — the same rule the - `transform` skill states for transforms, and for the same reasons: the row, its `entity_id` and - its YAML filename all survive. - - Tests live with the transform: they serialize and git-sync with it. Give each expectation a name that states the invariant, because the name is what a + Give each expectation a name that states the invariant, because the name is what a failure leads with ("revenue never negative", not "check 3"), and open each `empty` expectation's SQL with a `--` contract comment: the invariant, and the failure modes it catches ("catches both dropped orders and join fan-out"). @@ -104,10 +88,10 @@ tables for anomalies. ## Severity: error, or the tolerated oddity -Every expectation is pass or fail, and one failure fails the run — `run` exits non-zero, so a test -is a CI gate. **error** = forbidden, and it becomes an `empty` expectation. There is no warn -severity, so **never author an expectation you expect to fire**: a permanently red test trains -everyone to ignore the result and holds the gate down for everyone else. +Every expectation is pass or fail, and one failure fails the run. **error** = forbidden, and it +becomes an `empty` expectation. There is no warn severity, so **never author an expectation you +expect to fire**: a permanently red test trains everyone to ignore the result, and a red run gates +everyone else's work. A tolerated-but-surfaced oddity — one the owner lives with, like orphan rows or ship-before-order dates — gets encoded the two ways that hold: **pin it in the `equals` rows** (an orphan passing @@ -125,8 +109,8 @@ them to buggy output documents the bug as intended — and **surface the finding at both scales**, fixture ("1530.24 of 1600.74 fixture dollars survive") and warehouse ("908 of 2,050 orders dropped"). What happens next follows the session's terms, not a fixed protocol: propose and apply the fix now (when the user wants it or the autonomy setting covers it), or — when the fix -must wait — hold the correct expectation and record the red in the plan with the minimal fix body, -ready for `mb transform update --file`. A stored test has no red-by-design state, so a deferred +must wait — hold the correct expectation and record the red in the plan with the minimal fix body. +A stored test has no red-by-design state, so a deferred fix must be visible in the plan or the test reads as broken. Either way, once green the test stays as the regression guard. @@ -182,8 +166,5 @@ unshipped orders silently dropped from revenue — is what the zero-case rows ex - Don't let a fixture cast go all-clean — no zero-case, no orphan, no dirty row proves the happy path and nothing else; the bugs live in the edges. - Don't write an expectation you expect to fail. -- Don't name a table in an `empty` expectation that is neither the transform's target nor a declared - input. Only those two are rewritten to temp tables; anything else is left exactly as written and - reads the real table. - Don't surface bare check-ids ("per C3…") to the user — name the check in plain words; the ids are for your cross-referencing, not their reading. diff --git a/packages/cli/skill-data/transform/SKILL.md b/packages/cli/skill-data/transform/SKILL.md index 9f805e15..fa047735 100644 --- a/packages/cli/skill-data/transform/SKILL.md +++ b/packages/cli/skill-data/transform/SKILL.md @@ -191,6 +191,8 @@ mb transform-test run --profile --json # exits non-zero u **`expectations`** — `type: "empty"` with the `sql` that must return no rows, or `type: "equals"`, which needs the same `format` split as an input (`"rows"` with `columns`/`rows`, or `"sql"` with a query). An `equals` without a `format` is refused. +**An `empty` query may only name the transform's target table and its declared input tables.** Those are rewritten to the run's temp tables; any other table you name is left exactly as written and reads the real one — the single way a test run can touch production data. + ```json { "transform_id": 1, From c0bc2d1072575272f38e1b0996619bac86cc271a Mon Sep 17 00:00:00 2001 From: Alexander Polyankin Date: Thu, 17 Sep 2026 20:35:45 -0400 Subject: [PATCH 8/8] Transform tests ship in v65 Co-Authored-By: Claude Opus 5 --- README.md | 2 +- packages/cli/skill-data/core/SKILL.md | 2 +- packages/cli/skill-data/transform/SKILL.md | 2 +- packages/cli/src/commands/transform-test/create.ts | 2 +- packages/cli/src/commands/transform-test/delete.ts | 2 +- packages/cli/src/commands/transform-test/get.ts | 2 +- packages/cli/src/commands/transform-test/list.ts | 2 +- packages/cli/src/commands/transform-test/run.ts | 2 +- packages/cli/src/commands/transform-test/update.ts | 2 +- packages/cli/src/runtime/command-help.test.ts | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index e1954a1e..95f87aab 100644 --- a/README.md +++ b/README.md @@ -381,7 +381,7 @@ mb transform-tag delete 5 --yes ## Transform tests -CRUD and run on `/api/transform-test`. Requires Metabase v64 or newer. A transform test pins a transform's behaviour without touching real data: every table the transform reads is replaced by an `input` fixture, the transform runs into a temp table, and each `expectation` checks that output. The temp tables are dropped when the run ends. +CRUD and run on `/api/transform-test`. Requires Metabase v65 or newer. A transform test pins a transform's behaviour without touching real data: every table the transform reads is replaced by an `input` fixture, the transform runs into a temp table, and each `expectation` checks that output. The temp tables are dropped when the run ends. An input names its `table` and carries either `format: "sql"` with a `sql` query or `format: "rows"` with `columns` (each with a `database_type` the warehouse accepts as a `CAST` target) and `rows`. diff --git a/packages/cli/skill-data/core/SKILL.md b/packages/cli/skill-data/core/SKILL.md index 27071b78..0870d11d 100644 --- a/packages/cli/skill-data/core/SKILL.md +++ b/packages/cli/skill-data/core/SKILL.md @@ -140,7 +140,7 @@ Only what `--help` does _not_ tell you: footguns and non-obvious behaviors. - **collection ``** accepts four forms only — positive int, `root`, `trash`, or a 21-char entity_id; anything else is a client-side `ConfigError`. `collection items` pages the server endpoint, pulling only as far as the output cap can show — read `has_more`/`next_offset` to continue. `collection tree` is **JSON-only** (`--format text` is rejected). A transform collection needs `collection create --namespace transforms`. - **setting set** parses the value as **strict JSON**: a string is `'"value"'` (inner quotes), booleans `true`/`false`, numbers bare. Wrong quoting silently errors — confirm with `setting get ` after. `setting get --json` works on every value type (wrapping bare-text responses into `{key, value}`). - **search vs. list.** For plain enumeration of cards/dashboards/collections use the dedicated `… list` verbs; reach for `search --models ` only for ranking against a query string or a cross-resource lookup. -- **transform.** Iterate with `transform update `, never `delete` + `create` (keeps the row, `entity_id`, materialized table, and YAML filename — avoids `_2` suffixes and noisy git history). `transform run` needs `--wait` (or `--sync`, which also waits for the output table to register and returns `target_table_id`) or you get only `{run_id, final:null}`. `transform-test run ` checks a transform against fixtures in temp tables — no real table is read — and exits non-zero unless it passes (v64+). (→ `transform`.) +- **transform.** Iterate with `transform update `, never `delete` + `create` (keeps the row, `entity_id`, materialized table, and YAML filename — avoids `_2` suffixes and noisy git history). `transform run` needs `--wait` (or `--sync`, which also waits for the output table to register and returns `target_table_id`) or you get only `{run_id, final:null}`. `transform-test run ` checks a transform against fixtures in temp tables — no real table is read — and exits non-zero unless it passes (v65+). (→ `transform`.) - **setup is one-shot.** `mb setup` walks `/api/setup` for a **fresh** instance only — errors against an already-configured one. Mostly for bootstrapping local / e2e instances. - **eid** translates a string entity id → numeric id: `mb eid --model --json`. Entity ids are NanoIDs that can start with `-`, which the positional form misreads as a flag (shell quotes don't help) — for those, use `--body '{"entity_ids":{"card":["-…"]}}'` (the id is a JSON string value, immune to flag parsing). - **library.** EE-only (`library` premium feature, v59+). The Library is a curated subtree (`library-data` "Data" + `library-metrics` "Metrics" under a `library` root): tables published to **Data** appear first in data pickers and rank up in search; metrics saved to **Metrics** are prioritized in nav, search, and the query builder. `library get` shows the Library and its Data/Metrics collection ids; `library create` provisions it (idempotent). `library publish --table-ids/--db-ids/--schemas` publishes tables into Data — it **resolves the Data collection itself and creates the Library if absent** (no collection id to find); each `--schemas` entry is `:` (e.g. `1:public`), not a bare name. `publish` cascades to upstream FK dependencies, `unpublish` to downstream dependents; both need **admin or data-analyst** (Curate alone won't publish) and exit **403** without write **and** query permission on every affected table. Publish status shows on the table: `table get`/`table list` carry `is_published` (`collection_id` under `--full`). Publish finished, analysis-ready tables — clean/combine via transforms first. Publishing does not put the Data collection in the git-sync scope: on a remote-sync instance, `mb git-sync add-collection ` is what makes exports carry the published tables' metadata (see the `git-sync` skill). diff --git a/packages/cli/skill-data/transform/SKILL.md b/packages/cli/skill-data/transform/SKILL.md index fa047735..993a9b4b 100644 --- a/packages/cli/skill-data/transform/SKILL.md +++ b/packages/cli/skill-data/transform/SKILL.md @@ -172,7 +172,7 @@ mb transform run "$ID" --wait --profile --json # → succeeded If you really must `create + delete` instead, do the `delete` **before** the first `git-sync export` so the failed entity never lands in git history — an export of a soft-failed state is noise that needs a follow-up cleanup commit. See `git-sync`, "Read state before mutating", for the ordering rule. -## Transform tests (v64+) +## Transform tests (v65+) A transform test replaces every table the transform reads with a fixture, runs it into a temp table, and checks that output. No real table is read or written. diff --git a/packages/cli/src/commands/transform-test/create.ts b/packages/cli/src/commands/transform-test/create.ts index d1a4bfaf..c1007332 100644 --- a/packages/cli/src/commands/transform-test/create.ts +++ b/packages/cli/src/commands/transform-test/create.ts @@ -11,7 +11,7 @@ export default defineMetabaseCommand({ meta: { name: "create", description: "Create a transform test from JSON" }, details: 'The JSON body needs a `transform_id`, a `name`, the `inputs` standing in for every table the transform reads, and the `expectations` checked against its output. An input names its `table` and carries either `format: "sql"` with a `sql` query or `format: "rows"` with `columns` and `rows`. An expectation is either `type: "empty"` with the `sql` that must return nothing, or `type: "equals"`, which takes the same `format` split as an input: `format: "rows"` with the `columns` and `rows` the output must hold, or `format: "sql"` with a query returning them. The body is closed: a test read back with `get --full` has to shed `id`, `entity_id`, `creator_id`, `created_at` and `updated_at` before it can be sent.', - capabilities: { minVersion: 64 }, + capabilities: { minVersion: 65 }, args: { ...outputFlags, ...profileFlag, diff --git a/packages/cli/src/commands/transform-test/delete.ts b/packages/cli/src/commands/transform-test/delete.ts index e057deeb..8edc6e50 100644 --- a/packages/cli/src/commands/transform-test/delete.ts +++ b/packages/cli/src/commands/transform-test/delete.ts @@ -5,7 +5,7 @@ import { defineMetabaseCommand } from "../runtime"; export default defineMetabaseCommand({ meta: { name: "delete", description: "Delete a transform test by id" }, - capabilities: { minVersion: 64 }, + capabilities: { minVersion: 65 }, args: { ...outputFlags, ...profileFlag, diff --git a/packages/cli/src/commands/transform-test/get.ts b/packages/cli/src/commands/transform-test/get.ts index b3872fca..d1e50b85 100644 --- a/packages/cli/src/commands/transform-test/get.ts +++ b/packages/cli/src/commands/transform-test/get.ts @@ -10,7 +10,7 @@ export default defineMetabaseCommand({ meta: { name: "get", description: "Get a transform test by id" }, details: "The compact form carries the id, transform, name and description. Pass --full for the `inputs` and `expectations` themselves, which is also the body to edit and send back to `update`.", - capabilities: { minVersion: 64 }, + capabilities: { minVersion: 65 }, args: { ...outputFlags, ...profileFlag, diff --git a/packages/cli/src/commands/transform-test/list.ts b/packages/cli/src/commands/transform-test/list.ts index 059e3c59..6ca266f6 100644 --- a/packages/cli/src/commands/transform-test/list.ts +++ b/packages/cli/src/commands/transform-test/list.ts @@ -12,7 +12,7 @@ export const TransformTestListEnvelope = listEnvelopeSchema(TransformTestCompact export default defineMetabaseCommand({ meta: { name: "list", description: "List transform tests" }, - capabilities: { minVersion: 64 }, + capabilities: { minVersion: 65 }, args: { ...outputFlags, ...listFlags, diff --git a/packages/cli/src/commands/transform-test/run.ts b/packages/cli/src/commands/transform-test/run.ts index f82a3ba6..104f1ad7 100644 --- a/packages/cli/src/commands/transform-test/run.ts +++ b/packages/cli/src/commands/transform-test/run.ts @@ -40,7 +40,7 @@ export default defineMetabaseCommand({ meta: { name: "run", description: "Run a transform test by id" }, details: "Runs the transform against temp tables built from the test's inputs, checks every expectation against its output, and drops the temp tables. Nothing reads or writes a real table. A failing expectation exits non-zero; --json reports what each expectation found, including the rows a comparison disagreed on.", - capabilities: { minVersion: 64 }, + capabilities: { minVersion: 65 }, args: { ...outputFlags, ...profileFlag, diff --git a/packages/cli/src/commands/transform-test/update.ts b/packages/cli/src/commands/transform-test/update.ts index 7b5699dd..1086face 100644 --- a/packages/cli/src/commands/transform-test/update.ts +++ b/packages/cli/src/commands/transform-test/update.ts @@ -12,7 +12,7 @@ export default defineMetabaseCommand({ meta: { name: "update", description: "Update a transform test from JSON" }, details: "Only the fields the body carries are patched. `inputs` and `expectations` replace what is stored rather than merging into it. The body is closed: a test read back with `get --full` has to shed `id`, `entity_id`, `creator_id`, `created_at` and `updated_at` before it can be sent.", - capabilities: { minVersion: 64 }, + capabilities: { minVersion: 65 }, args: { ...outputFlags, ...profileFlag, diff --git a/packages/cli/src/runtime/command-help.test.ts b/packages/cli/src/runtime/command-help.test.ts index e802cbc5..2b6328dd 100644 --- a/packages/cli/src/runtime/command-help.test.ts +++ b/packages/cli/src/runtime/command-help.test.ts @@ -458,7 +458,7 @@ const CONTENT_TRANSLATION_CAPABILITIES = { } as const; const TRANSFORM_CAPABILITIES = { minVersion: 59 } as const; const TRANSFORM_JOB_SET_ACTIVE_CAPABILITIES = { minVersion: 61 } as const; -const TRANSFORM_TEST_CAPABILITIES = { minVersion: 64 } as const; +const TRANSFORM_TEST_CAPABILITIES = { minVersion: 65 } as const; let cachedEntries: Promise | null = null;