diff --git a/services/actions/src/rpc/__tests__/enrichmentEntitlement.test.js b/services/actions/src/rpc/__tests__/enrichmentEntitlement.test.js index a67bc4bd..9a03d1af 100644 --- a/services/actions/src/rpc/__tests__/enrichmentEntitlement.test.js +++ b/services/actions/src/rpc/__tests__/enrichmentEntitlement.test.js @@ -9,6 +9,7 @@ import { } from "../../utils/defaultModels/enrichmentEntitlement.js"; const KEY = "test-only-entitlement-key-with-at-least-32-bytes"; +const BILLING_CONNECTION_ID = "11111111-1111-4111-8111-111111111111"; const NOW = new Date("2026-08-30T12:00:00.000Z"); const signedLease = (overrides = {}) => { @@ -20,6 +21,7 @@ const signedLease = (overrides = {}) => { issued_at: "2026-08-30T11:55:00.000Z", valid_until: "2026-08-30T12:25:00.000Z", products: ["ctx:day-archetype", "ctx:weather-archetype"], + billing_connection_id: BILLING_CONNECTION_ID, ...overrides, }; return { @@ -48,6 +50,7 @@ describe("enrichment entitlement reconciliation", () => { assert.equal(result.valid, true); assert.equal(result.enabled, true); assert.equal(result.lease.entitlement_revision, "7"); + assert.equal(result.lease.billing_connection_id, BILLING_CONNECTION_ID); }); it("fails closed on signature tampering, expiry, and account mismatch", async () => { @@ -55,6 +58,7 @@ describe("enrichment entitlement reconciliation", () => { { ...signedLease(), signature: "tampered" }, signedLease({ valid_until: "2026-08-30T11:59:59.000Z" }), signedLease({ account_partition: "other.is" }), + signedLease({ billing_connection_id: null }), ]) { const result = await resolveEnrichmentEntitlement("customer.is", config, { now: () => NOW, @@ -94,6 +98,7 @@ describe("enrichment entitlement reconciliation", () => { signature_version: "hmac-sha256-v1", signature: "opaque", products: ["ctx:day-archetype", "ctx:weather-archetype"], + billing_connection_id: BILLING_CONNECTION_ID, }, }); assert.deepEqual(merged.default_models, settings.default_models); diff --git a/services/actions/src/utils/defaultModels/enrichmentEntitlement.js b/services/actions/src/utils/defaultModels/enrichmentEntitlement.js index 7e501f27..6d3763dc 100644 --- a/services/actions/src/utils/defaultModels/enrichmentEntitlement.js +++ b/services/actions/src/utils/defaultModels/enrichmentEntitlement.js @@ -21,6 +21,7 @@ const canonicalPayload = (payload) => issued_at: payload?.issued_at, valid_until: payload?.valid_until, products: payload?.products, + billing_connection_id: payload?.billing_connection_id, }); const signaturesMatch = (left, right) => { @@ -64,7 +65,12 @@ export const resolveEnrichmentEntitlement = async ( typeof payload?.enabled !== "boolean" || typeof payload?.entitlement_revision !== "string" || !Array.isArray(payload?.products) || - payload.products.join(",") !== PRODUCTS.join(",") + payload.products.join(",") !== PRODUCTS.join(",") || + (payload.enabled + ? !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( + payload.billing_connection_id || "", + ) + : payload.billing_connection_id !== null) ) { return disabled("lease_malformed"); } @@ -96,6 +102,7 @@ export const resolveEnrichmentEntitlement = async ( signature_version: envelope.signature_version, signature: envelope.signature, products: [...payload.products], + billing_connection_id: payload.billing_connection_id, }, }; } catch { @@ -115,6 +122,7 @@ export const mergeEnrichmentEntitlement = (settings = {}, resolution) => { signature_version: null, signature: null, products: PRODUCTS, + billing_connection_id: null, reason: resolution.reason || "invalid_lease", }; return { diff --git a/services/cubejs/package-lock.json b/services/cubejs/package-lock.json index a46cc792..3da55320 100644 --- a/services/cubejs/package-lock.json +++ b/services/cubejs/package-lock.json @@ -4,6 +4,7 @@ "requires": true, "packages": { "": { + "hasInstallScript": true, "dependencies": { "@cubejs-backend/api-gateway": "1.7.30", "@cubejs-backend/athena-driver": "1.7.30", @@ -39,6 +40,7 @@ "apache-arrow": "^21.1.0", "body-parser": "^1.19.0", "express": "4.22.2", + "extract-zip": "file:scripts/extract-zip-disabled", "http-proxy-middleware": "3.0.7", "ioredis": "^5.3.2", "jose": "^6.2.1", @@ -4473,9 +4475,9 @@ } }, "node_modules/@cubejs-backend/cubestore-driver/node_modules/ws": { - "version": "7.5.10", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", - "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", + "version": "7.5.11", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.11.tgz", + "integrity": "sha512-zS54Oen9bITtp7kp2XM3AydrCIq1D+HwJOuH+c+e4LfpL/lotP5osijd+UoMnxwAam1GN8R4KtLAyIrIcBNpiA==", "license": "MIT", "engines": { "node": ">=8.3.0" @@ -4899,6 +4901,7 @@ "node": ">=20.0.0" } }, + "node_modules/@cubejs-backend/native/node_modules/@cubejs-infra/post-installer/node_modules/@cubejs-backend/shared/scripts/extract-zip-disabled": {}, "node_modules/@cubejs-backend/native/node_modules/@octokit/auth-token": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", @@ -5028,6 +5031,10 @@ "@octokit/openapi-types": "^24.2.0" } }, + "node_modules/@cubejs-backend/native/node_modules/extract-zip": { + "resolved": "node_modules/@cubejs-backend/native/node_modules/@cubejs-infra/post-installer/node_modules/@cubejs-backend/shared/scripts/extract-zip-disabled", + "link": true + }, "node_modules/@cubejs-backend/native/node_modules/uuid": { "version": "11.1.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", @@ -5345,9 +5352,9 @@ } }, "node_modules/@cubejs-backend/server-core/node_modules/ws": { - "version": "7.5.10", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", - "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", + "version": "7.5.11", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.11.tgz", + "integrity": "sha512-zS54Oen9bITtp7kp2XM3AydrCIq1D+HwJOuH+c+e4LfpL/lotP5osijd+UoMnxwAam1GN8R4KtLAyIrIcBNpiA==", "license": "MIT", "engines": { "node": ">=8.3.0" @@ -5393,6 +5400,10 @@ "node": ">=20.0.0" } }, + "node_modules/@cubejs-backend/shared/node_modules/extract-zip": { + "resolved": "node_modules/@cubejs-backend/shared/scripts/extract-zip-disabled", + "link": true + }, "node_modules/@cubejs-backend/shared/node_modules/uuid": { "version": "11.1.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", @@ -5406,6 +5417,7 @@ "uuid": "dist/esm/bin/uuid" } }, + "node_modules/@cubejs-backend/shared/scripts/extract-zip-disabled": {}, "node_modules/@cubejs-backend/snowflake-driver": { "version": "1.7.30", "resolved": "https://registry.npmjs.org/@cubejs-backend/snowflake-driver/-/snowflake-driver-1.7.30.tgz", @@ -5545,6 +5557,27 @@ "node": "^12.0.0 || ^14.0.0 || >=16.0.0" } }, + "node_modules/@cubejs-backend/vertica-driver/node_modules/@cubejs-backend/cubestore-driver/node_modules/ws": { + "version": "7.5.11", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.11.tgz", + "integrity": "sha512-zS54Oen9bITtp7kp2XM3AydrCIq1D+HwJOuH+c+e4LfpL/lotP5osijd+UoMnxwAam1GN8R4KtLAyIrIcBNpiA==", + "license": "MIT", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/@cubejs-backend/vertica-driver/node_modules/@cubejs-backend/query-orchestrator": { "version": "0.31.69", "resolved": "https://registry.npmjs.org/@cubejs-backend/query-orchestrator/-/query-orchestrator-0.31.69.tgz", @@ -5755,27 +5788,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/@cubejs-backend/vertica-driver/node_modules/ws": { - "version": "7.5.13", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.13.tgz", - "integrity": "sha512-rsKI6xDBFVf4r/x8XyChGK04QR/XHroxs/jUcoWvtEZM8TPU/X/uIY9B1CsSzYws9ZJb/6bbBu7dPhFW00CAoA==", - "license": "MIT", - "engines": { - "node": ">=8.3.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/@dabh/diagnostics": { "version": "2.0.8", "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.8.tgz", @@ -7099,14 +7111,15 @@ } }, "node_modules/@types/request/node_modules/form-data": { - "version": "2.5.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.5.tgz", - "integrity": "sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==", + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.6.tgz", + "integrity": "sha512-Ogz/E85h9tlfJzpI6TuFpGcHZFhLrb9Gw8wq9v40CxSCPnv7ahKr6Xgtkn0KYCDQJ8DNn5VoMO8EXr9V5PadyA==", + "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", + "hasown": "^2.0.4", "mime-types": "^2.1.35", "safe-buffer": "^5.2.1" }, @@ -7125,16 +7138,6 @@ "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", "license": "MIT" }, - "node_modules/@types/yauzl": { - "version": "2.10.3", - "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", - "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", - "license": "MIT", - "optional": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@typespec/ts-http-runtime": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.1.tgz", @@ -7691,7 +7694,8 @@ "node_modules/async-limiter": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", - "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" + "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==", + "license": "MIT" }, "node_modules/async-mutex": { "version": "0.3.2", @@ -8037,9 +8041,10 @@ "integrity": "sha512-z4rE2Gxh7tvshQ4hluIT7XcFrgLIQaw9X3A+kTTRdovCz5PMukm/0QC/BKSYPj3omF5Qfypn9O/c5kgpmvYUCw==" }, "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -8121,14 +8126,6 @@ "ieee754": "^1.1.13" } }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "engines": { - "node": "*" - } - }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -9663,39 +9660,8 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "node_modules/extract-zip": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", - "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", - "license": "BSD-2-Clause", - "dependencies": { - "debug": "^4.1.1", - "get-stream": "^5.1.0", - "yauzl": "^2.10.0" - }, - "bin": { - "extract-zip": "cli.js" - }, - "engines": { - "node": ">= 10.17.0" - }, - "optionalDependencies": { - "@types/yauzl": "^2.9.1" - } - }, - "node_modules/extract-zip/node_modules/get-stream": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", - "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", - "license": "MIT", - "dependencies": { - "pump": "^3.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } + "resolved": "scripts/extract-zip-disabled", + "link": true }, "node_modules/fast-fifo": { "version": "1.3.2", @@ -9774,14 +9740,6 @@ "reusify": "^1.0.4" } }, - "node_modules/fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", - "dependencies": { - "pend": "~1.2.0" - } - }, "node_modules/fecha": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", @@ -11517,14 +11475,6 @@ "node": "*" } }, - "node_modules/jshs2/node_modules/commander": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.1.0.tgz", - "integrity": "sha512-J2wnb6TKniXNOtoHS8TSrG9IOQluPrsmyAJ8oCUJOBmv+uLBCyPYAZkD2jFvw2DCzIXNnISIM01NIvr35TkBMQ==", - "engines": { - "node": ">= 0.6.x" - } - }, "node_modules/jshs2/node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -11538,52 +11488,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, - "node_modules/jshs2/node_modules/nan": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-1.0.0.tgz", - "integrity": "sha512-Wm2/nFOm2y9HtJfgOLnctGbfvF23FcQZeyUZqDD8JQG3zO5kXh3MkQKiUaA68mJiVWrOzLFkAV1u6bC8P52DJA==" - }, - "node_modules/jshs2/node_modules/q": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/q/-/q-1.0.1.tgz", - "integrity": "sha512-18MnBaCeBX9sLRUdtxz/6onlb7wLzFxCylklyO8n27y5JxJYaGLPu4ccyc5zih58SpEzY8QmfwaWqguqXU6Y+A==", - "deprecated": "You or someone you depend on is using Q, the JavaScript Promise library that gave JavaScript developers strong feelings about promises. They can almost certainly migrate to the native JavaScript promise now. Thank you literally everyone for joining me in this bet against the odds. Be excellent to each other.\n\n(For a CapTP with native promises, see @endo/eventual-send and @endo/captp)", - "engines": { - "node": ">=0.6.0", - "teleport": ">=0.2.0" - } - }, - "node_modules/jshs2/node_modules/thrift": { - "version": "0.9.3", - "resolved": "https://registry.npmjs.org/thrift/-/thrift-0.9.3.tgz", - "integrity": "sha512-CLjTMnk31Yhcteeyfhxv1UU8yv0jqlc5kNDGogbpWG2mUU2sJtj33sQzA+qml8Kj2y8WUdVKY7kkZxiczKoATw==", - "dependencies": { - "node-int64": "~0.3.0", - "q": "1.0.x", - "ws": "~0.4.32" - }, - "engines": { - "node": ">= 0.2.4" - } - }, - "node_modules/jshs2/node_modules/ws": { - "version": "0.4.32", - "resolved": "https://registry.npmjs.org/ws/-/ws-0.4.32.tgz", - "integrity": "sha512-htqsS0U9Z9lb3ITjidQkRvkLdVhQePrMeu475yEfOWkAYvJ6dSjQp1tOH6ugaddzX5b7sQjMPNtY71eTzrV/kA==", - "hasInstallScript": true, - "dependencies": { - "commander": "~2.1.0", - "nan": "~1.0.0", - "options": ">=0.0.5", - "tinycolor": "0.x" - }, - "bin": { - "wscat": "bin/wscat" - }, - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -12804,14 +12708,6 @@ } } }, - "node_modules/options": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz", - "integrity": "sha512-bOj3L1ypm++N+n7CEbbe473A414AB7z+amKYshRb//iuL3MpdDCLhPnw6aVTdKB9g5ZRVHIEp8eUln6L2NUStg==", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/oracledb": { "version": "6.10.0", "resolved": "https://registry.npmjs.org/oracledb/-/oracledb-6.10.0.tgz", @@ -13123,9 +13019,10 @@ "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "license": "MIT", "engines": { "node": ">=8.6" }, @@ -13678,9 +13575,10 @@ } }, "node_modules/rimraf/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -13707,9 +13605,10 @@ } }, "node_modules/rimraf/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" }, @@ -13939,9 +13838,10 @@ } }, "node_modules/shelljs/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -13968,9 +13868,10 @@ } }, "node_modules/shelljs/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" }, @@ -14713,9 +14614,9 @@ } }, "node_modules/sqlite3/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "license": "MIT", "optional": true, "dependencies": { @@ -15600,14 +15501,16 @@ } }, "node_modules/thrift": { - "version": "0.20.0", - "resolved": "https://registry.npmjs.org/thrift/-/thrift-0.20.0.tgz", - "integrity": "sha512-oSmJTaoIAGolpupVHFfsWcmdEKX81fcDI6ty0hhezzdgZvp0XyXgMe9+1YusI8Ahy0HK4n8jlNrkPjOPeHZjdQ==", + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/thrift/-/thrift-0.23.0.tgz", + "integrity": "sha512-j7F1ls8JogClU88Ta/pwD/OzYuiFeD6Z5GoWw7ip+jcDhcNYFKgfXYEsyLXYpiNfJO94fbLnITGxyfZ19YzA6Q==", + "license": "Apache-2.0", "dependencies": { "browser-or-node": "^1.2.1", "isomorphic-ws": "^4.0.1", "node-int64": "^0.4.0", "q": "^1.5.0", + "uuid": "^13.0.0", "ws": "^5.2.3" }, "engines": { @@ -15619,10 +15522,24 @@ "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==" }, + "node_modules/thrift/node_modules/uuid": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.2.tgz", + "integrity": "sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, "node_modules/thrift/node_modules/ws": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.4.tgz", - "integrity": "sha512-fFCejsuC8f9kOSu9FYaOw8CdO68O3h5v0lg4p74o8JqWpwTf9tniOD+nOB78aWoVSS6WptVUmDrp/KPsMVBWFQ==", + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.5.tgz", + "integrity": "sha512-G0gACQIjFmv7NqpaOAXEpe9nEtRYD6ZCy2Ip/B4EzR08qEwnf/wYJoQd3cjosPdnJsHkeh0j2tzOaIBZIOC1yA==", + "license": "MIT", "dependencies": { "async-limiter": "~1.0.0" } @@ -15681,14 +15598,6 @@ "safe-buffer": "~5.1.0" } }, - "node_modules/tinycolor": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/tinycolor/-/tinycolor-0.0.1.tgz", - "integrity": "sha512-+CorETse1kl98xg0WAzii8DTT4ABF4R3nquhrkIbVGcw1T8JYs5Gfx9xEfGINPUZGDj9C4BmOtuKeaTtuuRolg==", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/tinyglobby": { "version": "0.2.17", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", @@ -16108,9 +16017,10 @@ } }, "node_modules/vertica-nodejs/node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -16137,9 +16047,10 @@ } }, "node_modules/vertica-nodejs/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" }, @@ -16431,9 +16342,9 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/ws": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", - "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", "license": "MIT", "peer": true, "engines": { @@ -16537,15 +16448,6 @@ "url": "https://github.com/sponsors/eemeli" } }, - "node_modules/yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", - "dependencies": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } - }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", @@ -16565,6 +16467,10 @@ "funding": { "url": "https://github.com/sponsors/colinhacks" } + }, + "scripts/extract-zip-disabled": { + "name": "extract-zip", + "version": "0.0.0-disabled" } } } diff --git a/services/cubejs/package.json b/services/cubejs/package.json index 6e11d9de..9f75518b 100644 --- a/services/cubejs/package.json +++ b/services/cubejs/package.json @@ -2,7 +2,7 @@ "private": true, "type": "module", "scripts": { - "postinstall": "node scripts/patchCubeYamlCompiler.mjs", + "postinstall": "node scripts/patchCubeYamlCompiler.mjs && node scripts/hardenRuntimeDependencies.mjs", "start": "node index.js", "start.dev": "nodemon --exitcrash --inspect=0.0.0.0 --max-old-space-size=8096 --max-http-header-size=32768 --watch src --watch index.js", "jsdoc": "jsdoc index.js -r src -d docs", @@ -44,6 +44,7 @@ "apache-arrow": "^21.1.0", "body-parser": "^1.19.0", "express": "4.22.2", + "extract-zip": "file:scripts/extract-zip-disabled", "http-proxy-middleware": "3.0.7", "ioredis": "^5.3.2", "jose": "^6.2.1", @@ -66,24 +67,36 @@ }, "overrides": { "@babel/plugin-transform-modules-systemjs": "7.29.4", + "brace-expansion@^1.0.0": "1.1.18", + "brace-expansion@^2.0.0": "2.1.4", "decompress": "npm:@xhmikosr/decompress@11.1.3", + "extract-zip": "$extract-zip", "fast-xml-parser": "5.11.1", + "form-data@^2.0.0": "2.5.6", "ip-address": "10.7.0", "joi": "17.13.4", "js-yaml": "4.3.1", "lodash": "4.18.1", + "minimatch@^3.0.0": "3.1.5", "path-to-regexp": "0.1.13", - "tar": "7.5.22" + "picomatch@^2.0.0": "2.3.2", + "tar": "7.5.22", + "thrift": "0.23.0", + "ws@^5.0.0": "5.2.5", + "ws@^7.0.0": "7.5.11", + "ws@^8.0.0": "8.21.0" }, "resolutions": { "@babel/plugin-transform-modules-systemjs": "7.29.4", "decompress": "npm:@xhmikosr/decompress@11.1.3", + "extract-zip": "file:scripts/extract-zip-disabled", "fast-xml-parser": "5.11.1", "ip-address": "10.7.0", "joi": "17.13.4", "js-yaml": "4.3.1", "lodash": "4.18.1", "path-to-regexp": "0.1.13", - "tar": "7.5.22" + "tar": "7.5.22", + "thrift": "0.23.0" } } diff --git a/services/cubejs/scripts/extract-zip-disabled/index.js b/services/cubejs/scripts/extract-zip-disabled/index.js new file mode 100644 index 00000000..11ad8146 --- /dev/null +++ b/services/cubejs/scripts/extract-zip-disabled/index.js @@ -0,0 +1,12 @@ +"use strict"; + +const disabled = () => { + const error = new Error( + "Archive extraction is disabled in the production runtime", + ); + error.code = "CUBE_RUNTIME_ARCHIVE_EXTRACTION_DISABLED"; + throw error; +}; + +module.exports = disabled; +module.exports.default = disabled; diff --git a/services/cubejs/scripts/extract-zip-disabled/package.json b/services/cubejs/scripts/extract-zip-disabled/package.json new file mode 100644 index 00000000..e3c3ccfd --- /dev/null +++ b/services/cubejs/scripts/extract-zip-disabled/package.json @@ -0,0 +1,7 @@ +{ + "name": "extract-zip", + "version": "0.0.0-disabled", + "private": true, + "main": "index.js", + "description": "Fail-closed runtime shim; archive extraction is not a Cube runtime capability" +} diff --git a/services/cubejs/scripts/hardenRuntimeDependencies.mjs b/services/cubejs/scripts/hardenRuntimeDependencies.mjs new file mode 100644 index 00000000..c87ab4f7 --- /dev/null +++ b/services/cubejs/scripts/hardenRuntimeDependencies.mjs @@ -0,0 +1,83 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const nodeModules = path.join(root, "node_modules"); + +async function readJson(file) { + return JSON.parse(await fs.readFile(file, "utf8")); +} + +async function findPackages(directory, packageName, found = []) { + for (const entry of await fs.readdir(directory, { withFileTypes: true })) { + if (!entry.isDirectory() || entry.name === ".bin") continue; + const child = path.join(directory, entry.name); + if (entry.name.startsWith("@")) { + await findPackages(child, packageName, found); + continue; + } + if (entry.name === packageName) { + const manifest = path.join(child, "package.json"); + try { + const parsed = await readJson(manifest); + if (parsed.name === packageName) found.push({ directory: child, parsed }); + } catch { + // A non-package directory with the same basename is irrelevant. + } + } + if (entry.name !== packageName) { + const nested = path.join(child, "node_modules"); + try { + if ((await fs.stat(nested)).isDirectory()) { + await findPackages(nested, packageName, found); + } + } catch { + // Most packages do not own a nested node_modules directory. + } + } + } + return found; +} + +const thriftPackages = await findPackages(nodeModules, "thrift"); +if (!thriftPackages.length || thriftPackages.some(({ parsed }) => parsed.version !== "0.23.0")) { + throw new Error("Every installed Apache Thrift package must resolve to 0.23.0"); +} + +const extractDirectory = path.join(nodeModules, "extract-zip"); +const extractManifest = path.join(extractDirectory, "package.json"); +const currentExtract = await readJson(extractManifest); + +if (currentExtract.version !== "0.0.0-disabled") { + if (currentExtract.name !== "extract-zip" || currentExtract.version !== "2.0.1") { + throw new Error("Refusing to replace an unreviewed extract-zip package"); + } + await fs.rm(extractDirectory, { recursive: true, force: false }); + await fs.mkdir(extractDirectory, { recursive: false }); + await fs.writeFile( + extractManifest, + `${JSON.stringify({ + name: "extract-zip", + version: "0.0.0-disabled", + private: true, + main: "index.js", + description: "Fail-closed runtime shim; archive extraction is install-time only", + }, null, 2)}\n`, + ); + await fs.writeFile( + path.join(extractDirectory, "index.js"), + `"use strict";\n` + + `const disabled = () => {\n` + + ` const error = new Error("Archive extraction is disabled in the production runtime");\n` + + ` error.code = "CUBE_RUNTIME_ARCHIVE_EXTRACTION_DISABLED";\n` + + ` throw error;\n` + + `};\n` + + `module.exports = disabled;\n` + + `module.exports.default = disabled;\n`, + ); +} + +console.log( + `Runtime dependency hardening verified (${thriftPackages.length} Thrift resolution(s), archive extraction disabled)`, +); diff --git a/services/cubejs/src/__tests__/runtimeDependencyHardening.test.js b/services/cubejs/src/__tests__/runtimeDependencyHardening.test.js new file mode 100644 index 00000000..08cf0741 --- /dev/null +++ b/services/cubejs/src/__tests__/runtimeDependencyHardening.test.js @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import { describe, it } from "node:test"; + +const require = createRequire(import.meta.url); + +describe("production dependency hardening", () => { + it("resolves the direct and Hive-transitive Thrift clients to 0.23.0", () => { + assert.equal(require("thrift/package.json").version, "0.23.0"); + + const hiveDirectory = require.resolve("@cubejs-backend/hive-driver/package.json"); + const hiveRequire = createRequire(hiveDirectory); + assert.equal(hiveRequire("thrift/package.json").version, "0.23.0"); + + const jshs2Directory = require.resolve("jshs2/package.json"); + const jshs2Require = createRequire(jshs2Directory); + assert.equal(jshs2Require("thrift/package.json").version, "0.23.0"); + }); + + it("constructs and releases the Hive driver with the hardened Thrift client", async () => { + const { default: HiveDriver } = await import("@cubejs-backend/hive-driver"); + const driver = new HiveDriver({ + host: "127.0.0.1", + port: 10000, + username: "test", + password: "test", + dbName: "default", + }); + + assert.equal(typeof driver.query, "function"); + assert.equal(typeof driver.testConnection, "function"); + assert.equal(HiveDriver.getDefaultConcurrency(), 2); + await driver.release(); + }); + + it("replaces install-time archive extraction with a fail-closed runtime shim", () => { + assert.equal(require("extract-zip/package.json").version, "0.0.0-disabled"); + const extract = require("extract-zip"); + assert.throws( + () => extract("untrusted.zip", { dir: "/tmp/never" }), + (error) => error?.code === "CUBE_RUNTIME_ARCHIVE_EXTRACTION_DISABLED", + ); + }); +}); diff --git a/services/cubejs/src/routes/__tests__/enrichmentGuards.test.js b/services/cubejs/src/routes/__tests__/enrichmentGuards.test.js index 8eadf93e..4e527b88 100644 --- a/services/cubejs/src/routes/__tests__/enrichmentGuards.test.js +++ b/services/cubejs/src/routes/__tests__/enrichmentGuards.test.js @@ -9,6 +9,7 @@ import { authorizeRunSqlQuery } from "../runSql.js"; const KEY = "test-enrichment-signing-key-at-least-32-bytes"; const NOW = new Date("2026-08-30T10:00:00.000Z"); const PRODUCTS = ["ctx:day-archetype", "ctx:weather-archetype"]; +const BILLING_CONNECTION_ID = "11111111-1111-4111-8111-111111111111"; function securityContext() { const payload = { @@ -19,6 +20,7 @@ function securityContext() { issued_at: "2026-08-30T09:00:00.000Z", valid_until: "2026-08-30T11:00:00.000Z", products: PRODUCTS, + billing_connection_id: BILLING_CONNECTION_ID, }; const signature = createHmac("sha256", KEY) .update(JSON.stringify(payload)) @@ -34,6 +36,7 @@ function securityContext() { issued_at: payload.issued_at, valid_until: payload.valid_until, products: payload.products, + billing_connection_id: payload.billing_connection_id, signature_version: "hmac-sha256-v1", signature, }, diff --git a/services/cubejs/src/utils/__tests__/enrichmentEntitlement.test.js b/services/cubejs/src/utils/__tests__/enrichmentEntitlement.test.js index 8ba3a4c5..3051e835 100644 --- a/services/cubejs/src/utils/__tests__/enrichmentEntitlement.test.js +++ b/services/cubejs/src/utils/__tests__/enrichmentEntitlement.test.js @@ -16,6 +16,7 @@ import { filterUnentitledEnrichmentSchemas } from "../repositoryFactory.js"; const KEY = "test-enrichment-signing-key-at-least-32-bytes"; const PRODUCTS = ["ctx:day-archetype", "ctx:weather-archetype"]; +const BILLING_CONNECTION_ID = "11111111-1111-4111-8111-111111111111"; const canonicalPayload = (payload) => JSON.stringify({ @@ -26,6 +27,7 @@ const canonicalPayload = (payload) => issued_at: payload.issued_at, valid_until: payload.valid_until, products: payload.products, + billing_connection_id: payload.billing_connection_id, }); function makeSecurityContext(overrides = {}) { @@ -37,6 +39,7 @@ function makeSecurityContext(overrides = {}) { issued_at: "2026-08-30T09:00:00.000Z", valid_until: "2026-08-30T11:00:00.000Z", products: PRODUCTS, + billing_connection_id: BILLING_CONNECTION_ID, ...overrides, }; const signature = createHmac("sha256", KEY) @@ -55,6 +58,7 @@ function makeSecurityContext(overrides = {}) { signature_version: "hmac-sha256-v1", signature, products: payload.products, + billing_connection_id: payload.billing_connection_id, }, }, }, @@ -111,12 +115,12 @@ describe("resolved enrichment member guard", () => { }); it("accepts a current, correctly signed lease", () => { - assert.equal( + assert.deepEqual( validateEnrichmentLease(makeSecurityContext(), { signingKey: KEY, now: NOW, - }).valid, - true, + }), + { valid: true, connectionId: BILLING_CONNECTION_ID }, ); assert.doesNotThrow(() => assertEnrichmentQueryAuthorized( @@ -133,9 +137,10 @@ describe("resolved enrichment member guard", () => { makeSecurityContext({ valid_until: "2026-08-30T09:59:59.000Z" }), makeSecurityContext({ enabled: false }), makeSecurityContext({ products: [PRODUCTS[0]] }), + makeSecurityContext({ billing_connection_id: null }), makeSecurityContext(), ]; - cases[4].userScope.teamProperties.premium.enrichment.signature = "invalid"; + cases[5].userScope.teamProperties.premium.enrichment.signature = "invalid"; for (const securityContext of cases) { assert.throws( diff --git a/services/cubejs/src/utils/__tests__/enrichmentMetering.test.js b/services/cubejs/src/utils/__tests__/enrichmentMetering.test.js index 37755a14..3a68f415 100644 --- a/services/cubejs/src/utils/__tests__/enrichmentMetering.test.js +++ b/services/cubejs/src/utils/__tests__/enrichmentMetering.test.js @@ -1,7 +1,11 @@ import assert from "node:assert/strict"; +import { createHmac } from "node:crypto"; import { describe, it } from "node:test"; -import { collectResolvedMembers } from "../enrichmentEntitlement.js"; +import { + collectResolvedMembers, + validateEnrichmentLease, +} from "../enrichmentEntitlement.js"; import { buildEnrichmentBillingBatch, deterministicBillingMessageId, @@ -12,10 +16,41 @@ import { resolveEnrichmentCodePrice, } from "../enrichmentPricing.js"; +const KEY = "test-enrichment-signing-key-at-least-32-bytes"; +const NOW = new Date("2026-08-30T10:00:00.000Z"); +const BILLING_CONNECTION_ID = "11111111-1111-4111-8111-111111111111"; +const LEASE_PAYLOAD = { + schema_version: 1, + account_partition: "tenant.is", + enabled: true, + entitlement_revision: "7", + issued_at: "2026-08-30T09:00:00.000Z", + valid_until: "2026-08-30T11:00:00.000Z", + products: ["ctx:day-archetype", "ctx:weather-archetype"], + billing_connection_id: BILLING_CONNECTION_ID, +}; const SECURITY_CONTEXT = { userId: "person-1", tokenPayload: { accountId: "account-real-1", partition: "tenant.is" }, - userScope: { teamProperties: { partition: "tenant.is" } }, + userScope: { + teamProperties: { + partition: "tenant.is", + premium: { + enrichment: { + enabled: LEASE_PAYLOAD.enabled, + entitlement_revision: LEASE_PAYLOAD.entitlement_revision, + issued_at: LEASE_PAYLOAD.issued_at, + valid_until: LEASE_PAYLOAD.valid_until, + products: LEASE_PAYLOAD.products, + billing_connection_id: LEASE_PAYLOAD.billing_connection_id, + signature_version: "hmac-sha256-v1", + signature: createHmac("sha256", KEY) + .update(JSON.stringify(LEASE_PAYLOAD)) + .digest("base64url"), + }, + }, + }, + }, }; const PRICING = Object.freeze({ pricingCodeVersion: "ctx-pricing-v1", @@ -23,12 +58,10 @@ const PRICING = Object.freeze({ "ctx:day-archetype": Object.freeze({ amount: "0.10", currency: "ISK", - connectionId: "connection-day-real", }), "ctx:weather-archetype": Object.freeze({ amount: "0.25", currency: "ISK", - connectionId: "connection-weather-real", }), }), }); @@ -60,6 +93,8 @@ const build = (req, result, options = {}) => buildEnrichmentBillingBatch(req, result, { resolveItems, resolvePrice, + validateLease: (securityContext) => + validateEnrichmentLease(securityContext, { signingKey: KEY, now: NOW }), ...options, }); @@ -175,6 +210,17 @@ describe("enrichment result-commit metering", () => { tokenPayload: { accountId: null, partition: "tenant.is" }, }; await assert.rejects(build(req, response([])), /real Account/); + + const missingConnection = structuredClone(SECURITY_CONTEXT); + missingConnection.userScope.teamProperties.premium.enrichment.billing_connection_id = null; + const missingConnectionRequest = request({ + dimensions: ["CtxDayContext.isHoliday"], + }); + missingConnectionRequest.context.securityContext = missingConnection; + await assert.rejects( + build(missingConnectionRequest, response([])), + /per-account enrichment billing Connection/, + ); }); it("does not commit a result when durable enqueue fails", async () => { @@ -235,7 +281,6 @@ describe("enrichment result-commit metering", () => { "ctx:day-archetype": { amount: "0.10", currency: "isk", - connection_id: "connection-day-real", }, }, }), @@ -250,5 +295,23 @@ describe("enrichment result-commit metering", () => { assert.equal(price.pricingResolution.pricingSource, "legacy_runtime_rate"); assert.equal(price.pricingCodeVersion, "ctx-pricing-v1"); assert.equal(price.unitAmount, "0.10"); + assert.equal("connectionId" in price, false); + }); + + it("attributes every enrichment item to the Connection in the signed account lease", async () => { + const result = await build( + request({ + dimensions: [ + "CtxDayContext.isHoliday", + "CtxWeatherContext.temperature", + ], + }), + response([]), + ); + assert.equal(result.length, 2); + assert.deepEqual( + result.map((entry) => entry.envelope.properties.connection_id), + [BILLING_CONNECTION_ID, BILLING_CONNECTION_ID], + ); }); }); diff --git a/services/cubejs/src/utils/enrichmentEntitlement.js b/services/cubejs/src/utils/enrichmentEntitlement.js index d1be2f03..3d8db6d6 100644 --- a/services/cubejs/src/utils/enrichmentEntitlement.js +++ b/services/cubejs/src/utils/enrichmentEntitlement.js @@ -28,6 +28,7 @@ const canonicalPayload = (payload) => issued_at: payload.issued_at, valid_until: payload.valid_until, products: payload.products, + billing_connection_id: payload.billing_connection_id, }); function signaturesMatch(left, right) { @@ -125,6 +126,9 @@ export function validateEnrichmentLease( typeof lease.entitlement_revision !== "string" || !Array.isArray(lease.products) || lease.products.join(",") !== ENRICHMENT_PRODUCTS.join(",") || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( + lease.billing_connection_id || "", + ) || !Number.isFinite(issuedAt) || !Number.isFinite(validUntil) || !Number.isFinite(nowMs) || @@ -142,11 +146,16 @@ export function validateEnrichmentLease( issued_at: lease.issued_at, valid_until: lease.valid_until, products: lease.products, + billing_connection_id: lease.billing_connection_id, }; const expected = createHmac("sha256", signingKey) .update(canonicalPayload(payload)) .digest("base64url"); - return { valid: signaturesMatch(expected, lease.signature) }; + const valid = signaturesMatch(expected, lease.signature); + return { + valid, + connectionId: valid ? lease.billing_connection_id : null, + }; } export function assertEnrichmentQueryAuthorized( diff --git a/services/cubejs/src/utils/enrichmentMetering.js b/services/cubejs/src/utils/enrichmentMetering.js index 50c2f332..f8efc6d0 100644 --- a/services/cubejs/src/utils/enrichmentMetering.js +++ b/services/cubejs/src/utils/enrichmentMetering.js @@ -5,7 +5,10 @@ import { recordBillingFailure } from "./billingMetrics.js"; import { buildConnectionCalled } from "./eventEmitter.js"; import { resolveEnrichmentCodePrice } from "./enrichmentPricing.js"; import { resolveEnrichmentBillingItems } from "./queryRewrite.js"; -import { sqlEnrichmentBillingItems } from "./enrichmentEntitlement.js"; +import { + sqlEnrichmentBillingItems, + validateEnrichmentLease, +} from "./enrichmentEntitlement.js"; function billingError(message) { const error = new Error(`enrichment billing unavailable: ${message}`); @@ -101,6 +104,7 @@ export async function buildEnrichmentBillingBatch( { resolveItems = resolveEnrichmentBillingItems, resolvePrice = resolveEnrichmentCodePrice, + validateLease = validateEnrichmentLease, surface = request?.apiType === "sql" ? "sql-api" : "rest", cacheStatus = "unknown", returnedRows = null, @@ -123,6 +127,7 @@ export async function buildEnrichmentBillingBatch( securityContext.userScope?.teamProperties?.partition || "", ).trim(); + let billingConnectionId = null; const entries = []; for (let index = 0; index < queries.length; index += 1) { @@ -131,6 +136,15 @@ export async function buildEnrichmentBillingBatch( if (!accountId || !partition) { throw billingError("real Account and tenant partition are required"); } + if (!billingConnectionId) { + const entitlement = validateLease(securityContext); + if (!entitlement?.valid || !entitlement.connectionId) { + throw billingError( + "a valid per-account enrichment billing Connection is required", + ); + } + billingConnectionId = entitlement.connectionId; + } const logicalExecutionId = logicalExecutionIdForRequest( request, queries.length > 1 ? index : null, @@ -144,7 +158,7 @@ export async function buildEnrichmentBillingBatch( userId: securityContext.userId || null, provider: "ctx", item, - connectionId: price.connectionId, + connectionId: billingConnectionId, billingMode: true, messageId, logicalExecutionId, diff --git a/services/cubejs/src/utils/enrichmentPricing.js b/services/cubejs/src/utils/enrichmentPricing.js index 29205464..9be9fb76 100644 --- a/services/cubejs/src/utils/enrichmentPricing.js +++ b/services/cubejs/src/utils/enrichmentPricing.js @@ -47,16 +47,12 @@ export function parseEnrichmentCodePricing( const currency = String(entry?.currency || "") .trim() .toUpperCase(); - const connectionId = String(entry?.connection_id || "").trim(); - if (!/^[A-Z]{3}$/.test(currency) || !connectionId) { - throw configurationError( - `${item} requires currency and a real Connection`, - ); + if (!/^[A-Z]{3}$/.test(currency)) { + throw configurationError(`${item} requires currency`); } items[item] = { amount: parseAmount(entry.amount), currency, - connectionId, }; } @@ -96,7 +92,6 @@ export function resolveEnrichmentCodePrice(item, options = {}) { ); } return { - connectionId: entry.connectionId, pricingCodeVersion: config.pricingCodeVersion, unitAmount: entry.amount, pricingResolution, diff --git a/services/cubejs/yarn.lock b/services/cubejs/yarn.lock index 34a146f8..506d8ff1 100644 --- a/services/cubejs/yarn.lock +++ b/services/cubejs/yarn.lock @@ -5455,16 +5455,8 @@ extend@^3.0.2: resolved "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== -extract-zip@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz" - integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== - dependencies: - debug "^4.1.1" - get-stream "^5.1.0" - yauzl "^2.10.0" - optionalDependencies: - "@types/yauzl" "^2.9.1" +extract-zip@^2.0.1, "extract-zip@file:scripts/extract-zip-disabled": + version "0.0.0-disabled" fast-fifo@^1.2.0, fast-fifo@^1.3.2: version "1.3.2" @@ -8716,26 +8708,18 @@ textextensions@^2.5.0: resolved "https://registry.npmjs.org/textextensions/-/textextensions-2.6.0.tgz" integrity sha512-49WtAWS+tcsy93dRt6P0P3AMD2m5PvXRhuEA0kaXos5ZLlujtYmpmFsB+QvWUSxE1ZsstmYXfQ7L40+EcQgpAQ== -thrift@^0.20.0: - version "0.20.0" - resolved "https://registry.npmjs.org/thrift/-/thrift-0.20.0.tgz" - integrity sha512-oSmJTaoIAGolpupVHFfsWcmdEKX81fcDI6ty0hhezzdgZvp0XyXgMe9+1YusI8Ahy0HK4n8jlNrkPjOPeHZjdQ== +thrift@0.23.0, thrift@^0.20.0, thrift@^0.9.3: + version "0.23.0" + resolved "https://registry.yarnpkg.com/thrift/-/thrift-0.23.0.tgz#de1162a3a2355f983eff73161bae57bca3fa5ccf" + integrity sha512-j7F1ls8JogClU88Ta/pwD/OzYuiFeD6Z5GoWw7ip+jcDhcNYFKgfXYEsyLXYpiNfJO94fbLnITGxyfZ19YzA6Q== dependencies: browser-or-node "^1.2.1" isomorphic-ws "^4.0.1" node-int64 "^0.4.0" q "^1.5.0" + uuid "^13.0.0" ws "^5.2.3" -thrift@^0.9.3: - version "0.9.3" - resolved "https://registry.npmjs.org/thrift/-/thrift-0.9.3.tgz" - integrity sha512-CLjTMnk31Yhcteeyfhxv1UU8yv0jqlc5kNDGogbpWG2mUU2sJtj33sQzA+qml8Kj2y8WUdVKY7kkZxiczKoATw== - dependencies: - node-int64 "~0.3.0" - q "1.0.x" - ws "~0.4.32" - throttle-debounce@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-3.0.1.tgz"