fix(cube): preserve literal YAML metadata

.github/workflows/build-containers.yml

@@ -66,10 +66,18 @@ jobs:
               ["hasura-migrations"]="services/hasura/"
               ["hasura-backend-plus"]="scripts/containers/hasura-backend-plus/"
             )
+
+            if [ "${{ github.event_name }}" = "pull_request" ]; then
+              BASE_SHA="${{ github.event.pull_request.base.sha }}"
+              HEAD_SHA="${{ github.event.pull_request.head.sha }}"
+            else
+              BASE_SHA="${{ github.event.before }}"
+              HEAD_SHA="${{ github.sha }}"
+            fi
 
             CHANGED_SERVICES=""
             for service in "${!SERVICE_PATHS[@]}"; do
-              if git diff --name-only "${{ github.event.before }}..${{ github.sha }}" | grep -E -q "^(${SERVICE_PATHS[$service]})"; then
+              if git diff --name-only "$BASE_SHA..$HEAD_SHA" | grep -E -q "^(${SERVICE_PATHS[$service]})"; then
                 CHANGED_SERVICES="$CHANGED_SERVICES,$service"
               fi
             done
@@ -138,6 +146,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Log in to Docker Hub
+        if: github.event_name != 'pull_request'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USER }}
@@ -151,8 +160,9 @@ jobs:
           tags: |
             type=ref,event=branch
             type=ref,event=pr
-            type=sha,prefix={{branch}}-,format=short
+            type=sha,prefix={{branch}}-,format=short,enable=${{ github.event_name != 'pull_request' }}
             type=sha,format=short,prefix=
+            type=raw,value=build-${{ github.sha }}
             type=raw,value=latest,enable={{is_default_branch}}
 
       - name: Build and push Docker image
@@ -162,12 +172,31 @@ jobs:
           context: ${{ matrix.context }}
           file: ${{ matrix.dockerfile }}
           platforms: linux/amd64
-          push: ${{ github.event_name != 'pull_request' }}
+          load: true
+          push: false
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha,scope=${{ matrix.service }}
           cache-to: type=gha,mode=max,scope=${{ matrix.service }}
 
+      - name: Test Cube image
+        if: matrix.service == 'cubejs'
+        run: |
+          docker run --rm \
+            -e CUBE_RUNTIME_EXPECTED_VERSION=1.7.30 \
+            -v "$GITHUB_WORKSPACE/tests:/tests:ro" \
+            "${{ matrix.image }}:build-${{ github.sha }}" \
+            yarn test
+
+      - name: Push tested image tags
+        if: github.event_name != 'pull_request'
+        env:
+          IMAGE_TAGS: ${{ steps.meta.outputs.tags }}
+        run: |
+          while IFS= read -r tag; do
+            [ -n "$tag" ] && docker push "$tag"
+          done <<< "$IMAGE_TAGS"
+
       - name: Output build summary
         id: output
         run: |
@@ -274,4 +303,4 @@ jobs:
 
           echo "" >> $GITHUB_STEP_SUMMARY
           echo "---" >> $GITHUB_STEP_SUMMARY
-          echo "*Generated by [GitHub Actions](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*" >> $GITHUB_STEP_SUMMARY
+          echo "*Generated by [GitHub Actions](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*" >> $GITHUB_STEP_SUMMARY

services/cubejs/Dockerfile

@@ -27,6 +27,7 @@ ENV PATH=/app/node_modules/.bin:$PATH
 
 COPY yarn.lock /app/yarn.lock
 COPY package.json /app/package.json
+COPY scripts/ /app/scripts/
 
 RUN yarn --frozen-lockfile --network-timeout 100000
 

services/cubejs/package.json

@@ -2,6 +2,7 @@
   "private": true,
   "type": "module",
   "scripts": {
+    "postinstall": "node scripts/patchCubeYamlCompiler.mjs",
     "start": "node index.js",
     "start.dev": "nodemon --exitcrash --inspect=0.0.0.0  --max-old-space-size=8096 --max-http-header-size=32768 --watch src --watch index.js",
     "jsdoc": "jsdoc index.js -r src -d docs",

services/cubejs/scripts/checkLiveSchemas.mjs

@@ -0,0 +1,17 @@
+import { createRequire } from "node:module";
+
+import { getDataSources } from "../src/utils/dataSourceHelpers.js";
+import { checkLiveSchemaCompatibility } from "../src/utils/liveSchemaCompatibility.js";
+
+const require = createRequire(import.meta.url);
+const compilerVersion = require(
+  "@cubejs-backend/schema-compiler/package.json",
+).version;
+
+const dataSources = await getDataSources();
+const summary = await checkLiveSchemaCompatibility(dataSources);
+
+// This is the complete output contract. Never add tenant, datasource, schema,
+// model, query, or compiler-error values to this release-gate record.
+console.log(JSON.stringify({ compilerVersion, ...summary }));
+if (!summary.compatible) process.exitCode = 1;

services/cubejs/scripts/patchCubeYamlCompiler.mjs

@@ -0,0 +1,66 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { dirname, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+
+const require = createRequire(import.meta.url);
+const SUPPORTED_VERSION = "1.7.30";
+const COMPILER_PACKAGE = "@cubejs-backend/schema-compiler";
+
+const ORIGINAL = `        else if (typeof obj === 'string') {
+            let code = obj;
+            if (!CubeValidator_1.nonStringFields.has(propertyPath[propertyPath.length - 1])) {`;
+
+const PATCHED = `        else if (typeof obj === 'string') {
+            // Cube metadata is an arbitrary literal payload exposed to API consumers.
+            // Treating it as Python f-string source makes JSON-valued metadata look
+            // like an interpolation expression and rejects otherwise valid models.
+            // Jinja rendering has already happened before this transformation.
+            if (propertyPath.includes('meta')) {
+                return t.stringLiteral(obj);
+            }
+            let code = obj;
+            if (!CubeValidator_1.nonStringFields.has(propertyPath[propertyPath.length - 1])) {`;
+
+const occurrences = (source, value) => source.split(value).length - 1;
+
+export function patchCompilerSource(source) {
+  if (source.includes(PATCHED)) return { source, changed: false };
+  if (occurrences(source, ORIGINAL) !== 1) {
+    throw new Error(
+      "Refusing to patch Cube YAML compiler: expected source anchor was not found exactly once",
+    );
+  }
+  return { source: source.replace(ORIGINAL, PATCHED), changed: true };
+}
+
+export async function patchInstalledCompiler() {
+  const packageJsonPath = require.resolve(`${COMPILER_PACKAGE}/package.json`);
+  const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8"));
+  if (packageJson.version !== SUPPORTED_VERSION) {
+    throw new Error(
+      `Refusing to patch ${COMPILER_PACKAGE} ${packageJson.version}; expected ${SUPPORTED_VERSION}`,
+    );
+  }
+
+  const compilerPath = resolve(
+    dirname(packageJsonPath),
+    "dist/src/compiler/YamlCompiler.js",
+  );
+  const current = await readFile(compilerPath, "utf8");
+  const result = patchCompilerSource(current);
+  if (result.changed) await writeFile(compilerPath, result.source, "utf8");
+  return { compilerPath, changed: result.changed };
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(resolve(process.argv[1])).href
+) {
+  const result = await patchInstalledCompiler();
+  console.log(
+    result.changed
+      ? `Patched Cube ${SUPPORTED_VERSION} YAML metadata handling`
+      : `Cube ${SUPPORTED_VERSION} YAML metadata patch already applied`,
+  );
+}

services/cubejs/src/__tests__/cube17Regression.test.js

@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
 import { access, readFile } from "node:fs/promises";
 import { createRequire } from "node:module";
 import { describe, it } from "node:test";
+import { prepareCompiler } from "@cubejs-backend/schema-compiler";
 
 import { escapeCSVField } from "../utils/csvSerializer.js";
 import {
@@ -10,6 +11,7 @@ import {
 } from "../utils/enrichmentEntitlement.js";
 import { deterministicBillingMessageId } from "../utils/enrichmentMetering.js";
 import { validateFormat } from "../utils/formatValidator.js";
+import { patchCompilerSource } from "../../scripts/patchCubeYamlCompiler.mjs";
 
 const require = createRequire(import.meta.url);
 const runtimeVersion = require("@cubejs-backend/server-core/package.json").version;
@@ -84,6 +86,55 @@ describe("Cube 1.6.68 to 1.7.30 comparative corpus", () => {
     );
   });
 
+  it("preserves JSON-valued metadata as a literal string", async () => {
+    const value = JSON.stringify({
+      time_zone: "Atlantic/Reykjavik",
+      preferred_source: "sensor",
+      language: "is",
+    });
+    const content = `cubes:
+  - name: support_ticket_analysed
+    sql_table: support_ticket_analysed
+    meta:
+      lc_values:
+        - '${value}'
+    dimensions:
+      - name: id
+        sql: id
+        type: string
+        primary_key: true
+`;
+    const repository = {
+      dataSchemaFiles: async () => [
+        { fileName: "support_ticket_analysed.yml", content },
+      ],
+    };
+    const { compiler, metaTransformer } = prepareCompiler(repository, {});
+
+    await compiler.compile();
+
+    assert.equal(metaTransformer.cubes[0].config.meta.lc_values[0], value);
+  });
+
+  it("guards the compiler patch against upstream source drift", () => {
+    const source = `before
+        else if (typeof obj === 'string') {
+            let code = obj;
+            if (!CubeValidator_1.nonStringFields.has(propertyPath[propertyPath.length - 1])) {
+after`;
+    const first = patchCompilerSource(source);
+    assert.equal(first.changed, true);
+    assert.match(first.source, /propertyPath\.includes\('meta'\)/);
+    assert.deepEqual(patchCompilerSource(first.source), {
+      source: first.source,
+      changed: false,
+    });
+    assert.throws(
+      () => patchCompilerSource("unexpected compiler source"),
+      /expected source anchor was not found exactly once/,
+    );
+  });
+
   it("does not embed a comparison result in the test source", async () => {
     const source = await readFile(new URL(import.meta.url), "utf8");
     assert.doesNotMatch(source, /pass(?:ed)?\s*[:=]\s*(?:true|yes)/i);

services/cubejs/src/__tests__/liveSchemaCompatibility.test.js

@@ -0,0 +1,50 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+
+import { checkLiveSchemaCompatibility } from "../utils/liveSchemaCompatibility.js";
+
+const source = (name, code = "secret tenant model") => ({ name, code });
+const context = (...dataschemas) => ({
+  id: "secret-datasource-id",
+  branches: [{ versions: [{ dataschemas }] }],
+});
+
+describe("privacy-preserving live schema compatibility gate", () => {
+  it("compiles every active context and reports only aggregate counts", async () => {
+    const seen = [];
+    const summary = await checkLiveSchemaCompatibility(
+      [context(source("one.yml")), context(source("two.js"))],
+      {
+        compile: async (files) => seen.push(files),
+      },
+    );
+
+    assert.equal(seen.length, 2);
+    assert.deepEqual(summary, {
+      compatible: true,
+      contexts: 2,
+      files: 2,
+      yamlFiles: 1,
+      javascriptFiles: 1,
+      compiledContexts: 2,
+      failedContexts: 0,
+    });
+    assert.doesNotMatch(JSON.stringify(summary), /secret|one|two|datasource/);
+  });
+
+  it("counts failures without retaining compiler errors or tenant values", async () => {
+    const summary = await checkLiveSchemaCompatibility(
+      [context(source("private.yml", "private model contents"))],
+      {
+        compile: async () => {
+          throw new Error("private compiler error and schema contents");
+        },
+      },
+    );
+
+    assert.equal(summary.compatible, false);
+    assert.equal(summary.failedContexts, 1);
+    assert.equal(summary.compiledContexts, 0);
+    assert.doesNotMatch(JSON.stringify(summary), /private|compiler error/);
+  });
+});

services/cubejs/src/utils/liveSchemaCompatibility.js

@@ -0,0 +1,62 @@
+import { prepareCompiler } from "@cubejs-backend/schema-compiler";
+
+const activeSchemas = (dataSource) =>
+  dataSource?.branches?.[0]?.versions?.[0]?.dataschemas || [];
+
+const schemaFile = (schema) => ({
+  fileName: schema.name,
+  readOnly: true,
+  content: schema.code,
+});
+
+export async function compileSchemaFiles(files) {
+  const repository = { dataSchemaFiles: async () => files };
+  const { compiler } = prepareCompiler(repository, {
+    allowNodeRequire: true,
+    standalone: true,
+  });
+  await compiler.compile();
+}
+
+/**
+ * Compile every active datasource model set without returning identifiers,
+ * filenames, source text, or compiler messages. This is intended for an
+ * in-cluster release gate where tenant schemas must never be exported in
+ * diagnostics.
+ */
+export async function checkLiveSchemaCompatibility(
+  dataSources,
+  { compile = compileSchemaFiles } = {},
+) {
+  const summary = {
+    compatible: true,
+    contexts: 0,
+    files: 0,
+    yamlFiles: 0,
+    javascriptFiles: 0,
+    compiledContexts: 0,
+    failedContexts: 0,
+  };
+
+  for (const dataSource of dataSources || []) {
+    const schemas = activeSchemas(dataSource);
+    summary.contexts += 1;
+    summary.files += schemas.length;
+    summary.yamlFiles += schemas.filter((schema) =>
+      /\.ya?ml$/i.test(schema?.name || ""),
+    ).length;
+    summary.javascriptFiles += schemas.filter((schema) =>
+      /\.js$/i.test(schema?.name || ""),
+    ).length;
+
+    try {
+      await compile(schemas.map(schemaFile));
+      summary.compiledContexts += 1;
+    } catch {
+      summary.compatible = false;
+      summary.failedContexts += 1;
+    }
+  }
+
+  return summary;
+}