From 03bf119e123671555a68582b1bb2282e26200a87 Mon Sep 17 00:00:00 2001 From: stefanbaxter Date: Sun, 30 Aug 2026 16:11:31 +0000 Subject: [PATCH 1/3] fix(cube): preserve literal YAML metadata --- services/cubejs/Dockerfile | 1 + services/cubejs/package.json | 1 + services/cubejs/scripts/checkLiveSchemas.mjs | 17 +++++ .../cubejs/scripts/patchCubeYamlCompiler.mjs | 66 +++++++++++++++++++ .../src/__tests__/cube17Regression.test.js | 51 ++++++++++++++ .../__tests__/liveSchemaCompatibility.test.js | 50 ++++++++++++++ .../src/utils/liveSchemaCompatibility.js | 62 +++++++++++++++++ 7 files changed, 248 insertions(+) create mode 100644 services/cubejs/scripts/checkLiveSchemas.mjs create mode 100644 services/cubejs/scripts/patchCubeYamlCompiler.mjs create mode 100644 services/cubejs/src/__tests__/liveSchemaCompatibility.test.js create mode 100644 services/cubejs/src/utils/liveSchemaCompatibility.js diff --git a/services/cubejs/Dockerfile b/services/cubejs/Dockerfile index 75ce6847..d1c698fd 100644 --- a/services/cubejs/Dockerfile +++ b/services/cubejs/Dockerfile @@ -27,6 +27,7 @@ ENV PATH=/app/node_modules/.bin:$PATH COPY yarn.lock /app/yarn.lock COPY package.json /app/package.json +COPY scripts/ /app/scripts/ RUN yarn --frozen-lockfile --network-timeout 100000 diff --git a/services/cubejs/package.json b/services/cubejs/package.json index fd6e4b84..6e11d9de 100644 --- a/services/cubejs/package.json +++ b/services/cubejs/package.json @@ -2,6 +2,7 @@ "private": true, "type": "module", "scripts": { + "postinstall": "node scripts/patchCubeYamlCompiler.mjs", "start": "node index.js", "start.dev": "nodemon --exitcrash --inspect=0.0.0.0 --max-old-space-size=8096 --max-http-header-size=32768 --watch src --watch index.js", "jsdoc": "jsdoc index.js -r src -d docs", diff --git a/services/cubejs/scripts/checkLiveSchemas.mjs b/services/cubejs/scripts/checkLiveSchemas.mjs new file mode 100644 index 00000000..8f8f406b --- /dev/null +++ b/services/cubejs/scripts/checkLiveSchemas.mjs @@ -0,0 +1,17 @@ +import { createRequire } from "node:module"; + +import { getDataSources } from "../src/utils/dataSourceHelpers.js"; +import { checkLiveSchemaCompatibility } from "../src/utils/liveSchemaCompatibility.js"; + +const require = createRequire(import.meta.url); +const compilerVersion = require( + "@cubejs-backend/schema-compiler/package.json", +).version; + +const dataSources = await getDataSources(); +const summary = await checkLiveSchemaCompatibility(dataSources); + +// This is the complete output contract. Never add tenant, datasource, schema, +// model, query, or compiler-error values to this release-gate record. +console.log(JSON.stringify({ compilerVersion, ...summary })); +if (!summary.compatible) process.exitCode = 1; diff --git a/services/cubejs/scripts/patchCubeYamlCompiler.mjs b/services/cubejs/scripts/patchCubeYamlCompiler.mjs new file mode 100644 index 00000000..132dde58 --- /dev/null +++ b/services/cubejs/scripts/patchCubeYamlCompiler.mjs @@ -0,0 +1,66 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { dirname, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +const require = createRequire(import.meta.url); +const SUPPORTED_VERSION = "1.7.30"; +const COMPILER_PACKAGE = "@cubejs-backend/schema-compiler"; + +const ORIGINAL = ` else if (typeof obj === 'string') { + let code = obj; + if (!CubeValidator_1.nonStringFields.has(propertyPath[propertyPath.length - 1])) {`; + +const PATCHED = ` else if (typeof obj === 'string') { + // Cube metadata is an arbitrary literal payload exposed to API consumers. + // Treating it as Python f-string source makes JSON-valued metadata look + // like an interpolation expression and rejects otherwise valid models. + // Jinja rendering has already happened before this transformation. + if (propertyPath.includes('meta')) { + return t.stringLiteral(obj); + } + let code = obj; + if (!CubeValidator_1.nonStringFields.has(propertyPath[propertyPath.length - 1])) {`; + +const occurrences = (source, value) => source.split(value).length - 1; + +export function patchCompilerSource(source) { + if (source.includes(PATCHED)) return { source, changed: false }; + if (occurrences(source, ORIGINAL) !== 1) { + throw new Error( + "Refusing to patch Cube YAML compiler: expected source anchor was not found exactly once", + ); + } + return { source: source.replace(ORIGINAL, PATCHED), changed: true }; +} + +export async function patchInstalledCompiler() { + const packageJsonPath = require.resolve(`${COMPILER_PACKAGE}/package.json`); + const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8")); + if (packageJson.version !== SUPPORTED_VERSION) { + throw new Error( + `Refusing to patch ${COMPILER_PACKAGE} ${packageJson.version}; expected ${SUPPORTED_VERSION}`, + ); + } + + const compilerPath = resolve( + dirname(packageJsonPath), + "dist/src/compiler/YamlCompiler.js", + ); + const current = await readFile(compilerPath, "utf8"); + const result = patchCompilerSource(current); + if (result.changed) await writeFile(compilerPath, result.source, "utf8"); + return { compilerPath, changed: result.changed }; +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(resolve(process.argv[1])).href +) { + const result = await patchInstalledCompiler(); + console.log( + result.changed + ? `Patched Cube ${SUPPORTED_VERSION} YAML metadata handling` + : `Cube ${SUPPORTED_VERSION} YAML metadata patch already applied`, + ); +} diff --git a/services/cubejs/src/__tests__/cube17Regression.test.js b/services/cubejs/src/__tests__/cube17Regression.test.js index 4af33968..b28a94f4 100644 --- a/services/cubejs/src/__tests__/cube17Regression.test.js +++ b/services/cubejs/src/__tests__/cube17Regression.test.js @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import { access, readFile } from "node:fs/promises"; import { createRequire } from "node:module"; import { describe, it } from "node:test"; +import { prepareCompiler } from "@cubejs-backend/schema-compiler"; import { escapeCSVField } from "../utils/csvSerializer.js"; import { @@ -10,6 +11,7 @@ import { } from "../utils/enrichmentEntitlement.js"; import { deterministicBillingMessageId } from "../utils/enrichmentMetering.js"; import { validateFormat } from "../utils/formatValidator.js"; +import { patchCompilerSource } from "../../scripts/patchCubeYamlCompiler.mjs"; const require = createRequire(import.meta.url); const runtimeVersion = require("@cubejs-backend/server-core/package.json").version; @@ -84,6 +86,55 @@ describe("Cube 1.6.68 to 1.7.30 comparative corpus", () => { ); }); + it("preserves JSON-valued metadata as a literal string", async () => { + const value = JSON.stringify({ + time_zone: "Atlantic/Reykjavik", + preferred_source: "sensor", + language: "is", + }); + const content = `cubes: + - name: support_ticket_analysed + sql_table: support_ticket_analysed + meta: + lc_values: + - '${value}' + dimensions: + - name: id + sql: id + type: string + primary_key: true +`; + const repository = { + dataSchemaFiles: async () => [ + { fileName: "support_ticket_analysed.yml", content }, + ], + }; + const { compiler, metaTransformer } = prepareCompiler(repository, {}); + + await compiler.compile(); + + assert.equal(metaTransformer.cubes[0].config.meta.lc_values[0], value); + }); + + it("guards the compiler patch against upstream source drift", () => { + const source = `before + else if (typeof obj === 'string') { + let code = obj; + if (!CubeValidator_1.nonStringFields.has(propertyPath[propertyPath.length - 1])) { +after`; + const first = patchCompilerSource(source); + assert.equal(first.changed, true); + assert.match(first.source, /propertyPath\.includes\('meta'\)/); + assert.deepEqual(patchCompilerSource(first.source), { + source: first.source, + changed: false, + }); + assert.throws( + () => patchCompilerSource("unexpected compiler source"), + /expected source anchor was not found exactly once/, + ); + }); + it("does not embed a comparison result in the test source", async () => { const source = await readFile(new URL(import.meta.url), "utf8"); assert.doesNotMatch(source, /pass(?:ed)?\s*[:=]\s*(?:true|yes)/i); diff --git a/services/cubejs/src/__tests__/liveSchemaCompatibility.test.js b/services/cubejs/src/__tests__/liveSchemaCompatibility.test.js new file mode 100644 index 00000000..8ed3e79b --- /dev/null +++ b/services/cubejs/src/__tests__/liveSchemaCompatibility.test.js @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { checkLiveSchemaCompatibility } from "../utils/liveSchemaCompatibility.js"; + +const source = (name, code = "secret tenant model") => ({ name, code }); +const context = (...dataschemas) => ({ + id: "secret-datasource-id", + branches: [{ versions: [{ dataschemas }] }], +}); + +describe("privacy-preserving live schema compatibility gate", () => { + it("compiles every active context and reports only aggregate counts", async () => { + const seen = []; + const summary = await checkLiveSchemaCompatibility( + [context(source("one.yml")), context(source("two.js"))], + { + compile: async (files) => seen.push(files), + }, + ); + + assert.equal(seen.length, 2); + assert.deepEqual(summary, { + compatible: true, + contexts: 2, + files: 2, + yamlFiles: 1, + javascriptFiles: 1, + compiledContexts: 2, + failedContexts: 0, + }); + assert.doesNotMatch(JSON.stringify(summary), /secret|one|two|datasource/); + }); + + it("counts failures without retaining compiler errors or tenant values", async () => { + const summary = await checkLiveSchemaCompatibility( + [context(source("private.yml", "private model contents"))], + { + compile: async () => { + throw new Error("private compiler error and schema contents"); + }, + }, + ); + + assert.equal(summary.compatible, false); + assert.equal(summary.failedContexts, 1); + assert.equal(summary.compiledContexts, 0); + assert.doesNotMatch(JSON.stringify(summary), /private|compiler error/); + }); +}); diff --git a/services/cubejs/src/utils/liveSchemaCompatibility.js b/services/cubejs/src/utils/liveSchemaCompatibility.js new file mode 100644 index 00000000..945817c7 --- /dev/null +++ b/services/cubejs/src/utils/liveSchemaCompatibility.js @@ -0,0 +1,62 @@ +import { prepareCompiler } from "@cubejs-backend/schema-compiler"; + +const activeSchemas = (dataSource) => + dataSource?.branches?.[0]?.versions?.[0]?.dataschemas || []; + +const schemaFile = (schema) => ({ + fileName: schema.name, + readOnly: true, + content: schema.code, +}); + +export async function compileSchemaFiles(files) { + const repository = { dataSchemaFiles: async () => files }; + const { compiler } = prepareCompiler(repository, { + allowNodeRequire: true, + standalone: true, + }); + await compiler.compile(); +} + +/** + * Compile every active datasource model set without returning identifiers, + * filenames, source text, or compiler messages. This is intended for an + * in-cluster release gate where tenant schemas must never be exported in + * diagnostics. + */ +export async function checkLiveSchemaCompatibility( + dataSources, + { compile = compileSchemaFiles } = {}, +) { + const summary = { + compatible: true, + contexts: 0, + files: 0, + yamlFiles: 0, + javascriptFiles: 0, + compiledContexts: 0, + failedContexts: 0, + }; + + for (const dataSource of dataSources || []) { + const schemas = activeSchemas(dataSource); + summary.contexts += 1; + summary.files += schemas.length; + summary.yamlFiles += schemas.filter((schema) => + /\.ya?ml$/i.test(schema?.name || ""), + ).length; + summary.javascriptFiles += schemas.filter((schema) => + /\.js$/i.test(schema?.name || ""), + ).length; + + try { + await compile(schemas.map(schemaFile)); + summary.compiledContexts += 1; + } catch { + summary.compatible = false; + summary.failedContexts += 1; + } + } + + return summary; +} From 57664aeea20ae61efb5ef0a51e4b4bdcf1362d4a Mon Sep 17 00:00:00 2001 From: stefanbaxter Date: Sun, 30 Aug 2026 16:13:02 +0000 Subject: [PATCH 2/3] fix(ci): test Cube image before push --- .github/workflows/build-containers.yml | 35 +++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 7051949e..6190efac 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -66,10 +66,18 @@ jobs: ["hasura-migrations"]="services/hasura/" ["hasura-backend-plus"]="scripts/containers/hasura-backend-plus/" ) + + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + HEAD_SHA="${{ github.event.pull_request.head.sha }}" + else + BASE_SHA="${{ github.event.before }}" + HEAD_SHA="${{ github.sha }}" + fi CHANGED_SERVICES="" for service in "${!SERVICE_PATHS[@]}"; do - if git diff --name-only "${{ github.event.before }}..${{ github.sha }}" | grep -E -q "^(${SERVICE_PATHS[$service]})"; then + if git diff --name-only "$BASE_SHA..$HEAD_SHA" | grep -E -q "^(${SERVICE_PATHS[$service]})"; then CHANGED_SERVICES="$CHANGED_SERVICES,$service" fi done @@ -138,6 +146,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Log in to Docker Hub + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USER }} @@ -153,6 +162,7 @@ jobs: type=ref,event=pr type=sha,prefix={{branch}}-,format=short type=sha,format=short,prefix= + type=raw,value=build-${{ github.sha }} type=raw,value=latest,enable={{is_default_branch}} - name: Build and push Docker image @@ -162,12 +172,31 @@ jobs: context: ${{ matrix.context }} file: ${{ matrix.dockerfile }} platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} + load: true + push: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha,scope=${{ matrix.service }} cache-to: type=gha,mode=max,scope=${{ matrix.service }} + - name: Test Cube image + if: matrix.service == 'cubejs' + run: | + docker run --rm \ + -e CUBE_RUNTIME_EXPECTED_VERSION=1.7.30 \ + -v "$GITHUB_WORKSPACE/tests:/tests:ro" \ + "${{ matrix.image }}:build-${{ github.sha }}" \ + yarn test + + - name: Push tested image tags + if: github.event_name != 'pull_request' + env: + IMAGE_TAGS: ${{ steps.meta.outputs.tags }} + run: | + while IFS= read -r tag; do + [ -n "$tag" ] && docker push "$tag" + done <<< "$IMAGE_TAGS" + - name: Output build summary id: output run: | @@ -274,4 +303,4 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY echo "---" >> $GITHUB_STEP_SUMMARY - echo "*Generated by [GitHub Actions](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + echo "*Generated by [GitHub Actions](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*" >> $GITHUB_STEP_SUMMARY From 0282e014d9d74590cf0466d29ff179a88a8661b8 Mon Sep 17 00:00:00 2001 From: stefanbaxter Date: Sun, 30 Aug 2026 16:14:18 +0000 Subject: [PATCH 3/3] fix(ci): avoid empty PR image tag --- .github/workflows/build-containers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 6190efac..9bb375a6 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -160,7 +160,7 @@ jobs: tags: | type=ref,event=branch type=ref,event=pr - type=sha,prefix={{branch}}-,format=short + type=sha,prefix={{branch}}-,format=short,enable=${{ github.event_name != 'pull_request' }} type=sha,format=short,prefix= type=raw,value=build-${{ github.sha }} type=raw,value=latest,enable={{is_default_branch}}