From b26bebd0ffaff79c43299366e85e0319676fa576 Mon Sep 17 00:00:00 2001 From: Joe O'Hallaron Date: Mon, 28 Sep 2026 16:13:44 -0600 Subject: [PATCH 01/11] feat: github-ify pr and puppeteer checks --- .../immerse-puppeteer-automation.yml | 134 ++++++++++++++++++ .github/workflows/mapd-immerse-pr.yml | 70 +++++++++ 2 files changed, 204 insertions(+) create mode 100644 .github/workflows/immerse-puppeteer-automation.yml create mode 100644 .github/workflows/mapd-immerse-pr.yml diff --git a/.github/workflows/immerse-puppeteer-automation.yml b/.github/workflows/immerse-puppeteer-automation.yml new file mode 100644 index 0000000..d7726f5 --- /dev/null +++ b/.github/workflows/immerse-puppeteer-automation.yml @@ -0,0 +1,134 @@ +name: immerse-puppeteer-automation + +# Replaces the legacy Jenkins immerse-puppeteer-automation job. +# The UI suite requires a HeavyDB instance containing the test data documented +# in src/ui-tests/README-Puppeteer-UI-tests.md. + +on: + push: + branches: + - "**" + workflow_dispatch: + inputs: + ref: + description: "Branch or SHA to test" + type: string + default: "" + +permissions: + contents: read + +concurrency: + group: immerse-puppeteer-automation-${{ github.ref }} + cancel-in-progress: true + +jobs: + immerse-puppeteer-automation: + name: immerse-puppeteer-automation + runs-on: linux-amd64-cpu32 + timeout-minutes: 90 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + cache-dependency-path: package-lock.json + + - name: Upgrade npm + run: npm install --global npm@^11.6.1 + + - name: Use HTTPS for GitHub dependencies + run: | + git config --global --add \ + url."https://github.com/".insteadOf \ + "git@github.com:" + git config --global --add \ + url."https://github.com/".insteadOf \ + "ssh://git@github.com/" + + - name: Install dependencies + run: npm ci + + - name: Configure test server + env: + TEST_DATABASE: ${{ secrets.PLAYWRIGHT_TEST_DB }} + TEST_USERNAME: ${{ secrets.PLAYWRIGHT_TEST_USER }} + TEST_PASSWORD: ${{ secrets.PLAYWRIGHT_TEST_PASSWORD }} + TEST_SERVER: ${{ secrets.PLAYWRIGHT_TEST_SERVER }} + run: | + node <<'NODE' + const fs = require("fs") + + const required = [ + "TEST_DATABASE", + "TEST_USERNAME", + "TEST_PASSWORD", + "TEST_SERVER" + ] + const missing = required.filter((name) => !process.env[name]) + if (missing.length) { + throw new Error(`Missing required secrets: ${missing.join(", ")}`) + } + + const server = { + database: process.env.TEST_DATABASE, + master: true, + username: process.env.TEST_USERNAME, + password: process.env.TEST_PASSWORD, + url: process.env.TEST_SERVER, + GTM: "GTM-MDD888", + theme: "dark" + } + + fs.writeFileSync( + "src/servers.local.json", + `${JSON.stringify([server], null, 2)}\n` + ) + NODE + + - name: Start Immerse + run: | + npm run start -- --no-open > webpack.log 2>&1 & + echo "WEBPACK_SERVER_PID=$!" >> "$GITHUB_ENV" + + - name: Wait for Immerse + run: | + for attempt in {1..120}; do + if curl --fail --silent --show-error http://localhost:8002/ > /dev/null; then + exit 0 + fi + sleep 5 + done + + cat webpack.log + echo "Immerse did not become ready within 10 minutes" >&2 + exit 1 + + - name: Run Puppeteer integration tests + run: npm run test:ui + + - name: Stop Immerse + if: always() + run: | + if [ -n "${WEBPACK_SERVER_PID:-}" ]; then + kill "$WEBPACK_SERVER_PID" 2>/dev/null || true + fi + + - name: Upload test diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: puppeteer-test-diagnostics + path: | + jest-results.xml + webpack.log + src/ui-tests/_screenshot-*.png + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/mapd-immerse-pr.yml b/.github/workflows/mapd-immerse-pr.yml new file mode 100644 index 0000000..8d2ea4f --- /dev/null +++ b/.github/workflows/mapd-immerse-pr.yml @@ -0,0 +1,70 @@ +name: mapd-immerse-pr + +# Replaces the legacy Jenkins mapd-immerse-pr job. +# ESLint also enforces the repository's Prettier rules. + +on: + push: + branches: + - "**" + workflow_dispatch: + inputs: + ref: + description: "Branch or SHA to test" + type: string + default: "" + +permissions: + contents: read + +concurrency: + group: mapd-immerse-pr-${{ github.ref }} + cancel-in-progress: true + +jobs: + mapd-immerse-pr: + name: mapd-immerse-pr + runs-on: linux-amd64-cpu16 + timeout-minutes: 60 + env: + # The PR checks do not use a browser. Avoid downloading Chromium during + # npm ci; the Puppeteer workflow installs it independently. + PUPPETEER_SKIP_DOWNLOAD: "true" + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: "true" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + cache-dependency-path: package-lock.json + + - name: Upgrade npm + run: npm install --global npm@^11.6.1 + + - name: Use HTTPS for GitHub dependencies + run: | + git config --global --add \ + url."https://github.com/".insteadOf \ + "git@github.com:" + git config --global --add \ + url."https://github.com/".insteadOf \ + "ssh://git@github.com/" + + - name: Install dependencies + run: npm ci + + - name: Lint and check formatting + run: npm run lint + + - name: Run unit tests + run: npm run test:unit + + - name: Run component tests + run: npm run test:component From 38c29a26c973fcf005e88d8f1ab0863447503a38 Mon Sep 17 00:00:00 2001 From: Joe O'Hallaron Date: Mon, 28 Sep 2026 16:15:07 -0600 Subject: [PATCH 02/11] fix: remove mapd references --- .../workflows/{mapd-immerse-pr.yml => immerse-pr.yml} | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) rename .github/workflows/{mapd-immerse-pr.yml => immerse-pr.yml} (90%) diff --git a/.github/workflows/mapd-immerse-pr.yml b/.github/workflows/immerse-pr.yml similarity index 90% rename from .github/workflows/mapd-immerse-pr.yml rename to .github/workflows/immerse-pr.yml index 8d2ea4f..37a4b65 100644 --- a/.github/workflows/mapd-immerse-pr.yml +++ b/.github/workflows/immerse-pr.yml @@ -1,6 +1,5 @@ -name: mapd-immerse-pr +name: immerse-pr -# Replaces the legacy Jenkins mapd-immerse-pr job. # ESLint also enforces the repository's Prettier rules. on: @@ -18,12 +17,12 @@ permissions: contents: read concurrency: - group: mapd-immerse-pr-${{ github.ref }} + group: immerse-pr-${{ github.ref }} cancel-in-progress: true jobs: - mapd-immerse-pr: - name: mapd-immerse-pr + immerse-pr: + name: immerse-pr runs-on: linux-amd64-cpu16 timeout-minutes: 60 env: From 19c6cfa65c0693dc4fc848c0da512955a7fd7253 Mon Sep 17 00:00:00 2001 From: Joe O'Hallaron Date: Mon, 28 Sep 2026 16:33:21 -0600 Subject: [PATCH 03/11] fix: lint, prettier, unit test failures --- .eslintignore | 1 + scripts/generate-third-party-licenses.js | 172 ++++++++++-------- src/actions/connection-action-creators.js | 3 +- src/charts/bubble/scatter.unit.test.js | 7 +- src/charts/heat/heat.unit.test.js | 7 +- src/charts/pie/pie.unit.test.js | 7 +- src/charts/row/row.unit.test.js | 7 +- src/charts/table/table.unit.test.js | 7 +- src/charts/utils/event-listeners.unit.test.js | 10 - src/components/account-panel/account-panel.js | 2 +- .../chart-type-panel/chart-type-panel.js | 3 +- src/components/dashboards/dashboards.js | 152 ++++++++-------- src/components/data-source-selector/utils.js | 1 - src/components/login/login.tsx | 61 ++++--- .../navigation-bar-container.tsx | 1 - src/components/settings/settings-nav-item.tsx | 2 +- src/components/shared-settings/util.ts | 1 - .../categorical/categorical-summary-chart.tsx | 2 +- .../temporal/temporal-summary-chart.tsx | 4 +- .../sql-notebook/left-panel/left-panel.tsx | 27 +-- .../results/sql-notebook-answer-result.tsx | 1 - src/components/svg-icons/index.js | 1 - src/reducers/user-configurable-ui-reducer.ts | 2 +- src/vega/charts/combo-chart/combo-chart.ts | 3 +- src/vega/charts/combo-chart/row-spec.ts | 11 +- 25 files changed, 235 insertions(+), 260 deletions(-) diff --git a/.eslintignore b/.eslintignore index d44fd70..beb5b1f 100644 --- a/.eslintignore +++ b/.eslintignore @@ -8,3 +8,4 @@ frontend-dev/ frontend-prod/ node_modules/ playwright-report/ +src/polyfills/css-vars.js diff --git a/scripts/generate-third-party-licenses.js b/scripts/generate-third-party-licenses.js index 296b88a..4e1cc95 100644 --- a/scripts/generate-third-party-licenses.js +++ b/scripts/generate-third-party-licenses.js @@ -1,99 +1,117 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -const fs = require('fs'); -const path = require('path'); -const licenseChecker = require('license-checker'); +const fs = require("fs") +const path = require("path") +const licenseChecker = require("license-checker") -const OUTPUT_PATH = path.join(__dirname, '../third_party_licenses/THIRD_PARTY_LICENSES.md'); -const PROJECT_ROOT = path.join(__dirname, '..'); +const OUTPUT_PATH = path.join( + __dirname, + "../third_party_licenses/THIRD_PARTY_LICENSES.md" +) +const PROJECT_ROOT = path.join(__dirname, "..") const spdxUrls = { - '0BSD': 'https://spdx.org/licenses/0BSD.html', - 'Apache-2.0': 'https://spdx.org/licenses/Apache-2.0.html', - 'Apache 2.0': 'https://spdx.org/licenses/Apache-2.0.html', - 'Apache*': 'https://spdx.org/licenses/Apache-2.0.html', - 'Apache-Style': 'https://spdx.org/licenses/Apache-2.0.html', - '(Apache-2.0 OR MPL-1.1)': 'https://spdx.org/licenses/Apache-2.0.html', - 'Artistic-2.0': 'https://spdx.org/licenses/Artistic-2.0.html', - 'BSD': 'https://spdx.org/licenses/BSD-2-Clause.html', - 'BSD*': 'https://spdx.org/licenses/BSD-2-Clause.html', - 'BSD-2-Clause': 'https://spdx.org/licenses/BSD-2-Clause.html', - '(BSD-2-Clause OR WTFPL)': 'https://spdx.org/licenses/BSD-2-Clause.html', - 'BSD-3-Clause': 'https://spdx.org/licenses/BSD-3-Clause.html', - '(BSD-3-Clause OR GPL-2.0)': 'https://spdx.org/licenses/BSD-3-Clause.html', - '(AFL-2.1 OR BSD-3-Clause)': 'https://spdx.org/licenses/BSD-3-Clause.html', - 'BlueOak-1.0.0': 'https://spdx.org/licenses/BlueOak-1.0.0.html', - 'CC-BY-3.0': 'https://spdx.org/licenses/CC-BY-3.0.html', - 'CC-BY-4.0': 'https://spdx.org/licenses/CC-BY-4.0.html', - 'CC0-1.0': 'https://spdx.org/licenses/CC0-1.0.html', - 'ISC': 'https://spdx.org/licenses/ISC.html', - 'MIT': 'https://spdx.org/licenses/MIT.html', - 'MIT*': 'https://spdx.org/licenses/MIT.html', - 'MIT-0': 'https://spdx.org/licenses/MIT-0.html', - 'MIT, CC-BY-SA-4.0': 'https://spdx.org/licenses/MIT.html', - '(MIT AND BSD-3-Clause)': 'https://spdx.org/licenses/MIT.html', - '(MIT AND CC-BY-3.0)': 'https://spdx.org/licenses/MIT.html', - '(MIT AND Zlib)': 'https://spdx.org/licenses/MIT.html', - '(MIT OR Apache-2.0)': 'https://spdx.org/licenses/MIT.html', - '(MIT OR CC0-1.0)': 'https://spdx.org/licenses/MIT.html', - '(MIT OR GPL-3.0-or-later)': 'https://spdx.org/licenses/MIT.html', - '(WTFPL OR MIT)': 'https://spdx.org/licenses/MIT.html', - 'MPL-2.0': 'https://spdx.org/licenses/MPL-2.0.html', - 'Python-2.0': 'https://spdx.org/licenses/Python-2.0.html', - 'Unlicense': 'https://spdx.org/licenses/Unlicense.html', - '(Unlicense OR Apache-2.0)': 'https://spdx.org/licenses/Unlicense.html', - 'UNLICENSED': '', - 'WTFPL': 'https://spdx.org/licenses/WTFPL.html', - 'Public Domain': '', -}; + "0BSD": "https://spdx.org/licenses/0BSD.html", + "Apache-2.0": "https://spdx.org/licenses/Apache-2.0.html", + "Apache 2.0": "https://spdx.org/licenses/Apache-2.0.html", + "Apache*": "https://spdx.org/licenses/Apache-2.0.html", + "Apache-Style": "https://spdx.org/licenses/Apache-2.0.html", + "(Apache-2.0 OR MPL-1.1)": "https://spdx.org/licenses/Apache-2.0.html", + "Artistic-2.0": "https://spdx.org/licenses/Artistic-2.0.html", + BSD: "https://spdx.org/licenses/BSD-2-Clause.html", + "BSD*": "https://spdx.org/licenses/BSD-2-Clause.html", + "BSD-2-Clause": "https://spdx.org/licenses/BSD-2-Clause.html", + "(BSD-2-Clause OR WTFPL)": "https://spdx.org/licenses/BSD-2-Clause.html", + "BSD-3-Clause": "https://spdx.org/licenses/BSD-3-Clause.html", + "(BSD-3-Clause OR GPL-2.0)": "https://spdx.org/licenses/BSD-3-Clause.html", + "(AFL-2.1 OR BSD-3-Clause)": "https://spdx.org/licenses/BSD-3-Clause.html", + "BlueOak-1.0.0": "https://spdx.org/licenses/BlueOak-1.0.0.html", + "CC-BY-3.0": "https://spdx.org/licenses/CC-BY-3.0.html", + "CC-BY-4.0": "https://spdx.org/licenses/CC-BY-4.0.html", + "CC0-1.0": "https://spdx.org/licenses/CC0-1.0.html", + ISC: "https://spdx.org/licenses/ISC.html", + MIT: "https://spdx.org/licenses/MIT.html", + "MIT*": "https://spdx.org/licenses/MIT.html", + "MIT-0": "https://spdx.org/licenses/MIT-0.html", + "MIT, CC-BY-SA-4.0": "https://spdx.org/licenses/MIT.html", + "(MIT AND BSD-3-Clause)": "https://spdx.org/licenses/MIT.html", + "(MIT AND CC-BY-3.0)": "https://spdx.org/licenses/MIT.html", + "(MIT AND Zlib)": "https://spdx.org/licenses/MIT.html", + "(MIT OR Apache-2.0)": "https://spdx.org/licenses/MIT.html", + "(MIT OR CC0-1.0)": "https://spdx.org/licenses/MIT.html", + "(MIT OR GPL-3.0-or-later)": "https://spdx.org/licenses/MIT.html", + "(WTFPL OR MIT)": "https://spdx.org/licenses/MIT.html", + "MPL-2.0": "https://spdx.org/licenses/MPL-2.0.html", + "Python-2.0": "https://spdx.org/licenses/Python-2.0.html", + Unlicense: "https://spdx.org/licenses/Unlicense.html", + "(Unlicense OR Apache-2.0)": "https://spdx.org/licenses/Unlicense.html", + UNLICENSED: "", + WTFPL: "https://spdx.org/licenses/WTFPL.html", + "Public Domain": "" +} function getLicenseUrl(license, repository) { - if (typeof license !== 'string') return repository || ''; - if (license.startsWith('Custom: ')) { - return license.slice('Custom: '.length).trim(); + if (typeof license !== "string") { + return repository || "" + } + if (license.startsWith("Custom: ")) { + return license.slice("Custom: ".length).trim() + } + const mapped = spdxUrls[license] + if (mapped !== undefined) { + return mapped } - const mapped = spdxUrls[license]; - if (mapped !== undefined) return mapped; - return repository || ''; + return repository || "" } licenseChecker.init({ start: PROJECT_ROOT }, (err, data) => { if (err) { - console.error('Error running license-checker:', err); - process.exit(1); + throw err } const rows = Object.entries(data).map(([pkgAtVersion, info]) => { - const lastAt = pkgAtVersion.lastIndexOf('@'); - const name = pkgAtVersion.slice(0, lastAt); - const version = pkgAtVersion.slice(lastAt + 1); - const rawLicense = info.licenses; - const license = Array.isArray(rawLicense) ? rawLicense.join(', ') : (rawLicense || 'Unknown'); - const licenseUrl = getLicenseUrl(license, info.repository); - return { name, version, license, licenseUrl }; - }); + const lastAt = pkgAtVersion.lastIndexOf("@") + const name = pkgAtVersion.slice(0, lastAt) + const version = pkgAtVersion.slice(lastAt + 1) + const rawLicense = info.licenses + const license = Array.isArray(rawLicense) + ? rawLicense.join(", ") + : rawLicense || "Unknown" + const licenseUrl = getLicenseUrl(license, info.repository) + return { name, version, license, licenseUrl } + }) - rows.sort((a, b) => a.name.toLowerCase().localeCompare(b.name.toLowerCase())); + rows.sort((a, b) => a.name.toLowerCase().localeCompare(b.name.toLowerCase())) - const lines = []; - lines.push('# Third-Party Licenses'); - lines.push(''); - lines.push('This file lists the third-party npm packages distributed with this project (the runtime dependency closure of `package.json`) and their licenses, in fulfillment of the attribution requirements of those licenses.'); - lines.push(''); - lines.push('Generated from the installed environment. Total packages: **' + rows.length + '**.'); - lines.push(''); - lines.push('## Summary'); - lines.push(''); - lines.push('| Package | Version | License | URL |'); - lines.push('|---|---|---|---|'); + const lines = [] + lines.push("# Third-Party Licenses") + lines.push("") + lines.push( + "This file lists the third-party npm packages distributed with this project (the runtime dependency closure of `package.json`) and their licenses, in fulfillment of the attribution requirements of those licenses." + ) + lines.push("") + lines.push( + `Generated from the installed environment. Total packages: **${rows.length}**.` + ) + lines.push("") + lines.push("## Summary") + lines.push("") + lines.push("| Package | Version | License | URL |") + lines.push("|---|---|---|---|") for (const row of rows) { - const urlCell = row.licenseUrl ? `[${row.licenseUrl}](${row.licenseUrl})` : ''; - lines.push(`| ${row.name} | ${row.version} | ${row.license} | ${urlCell} |`); + const urlCell = row.licenseUrl + ? `[${row.licenseUrl}](${row.licenseUrl})` + : "" + lines.push(`| ${row.name} | ${row.version} | ${row.license} | ${urlCell} |`) } - const output = lines.join('\n') + '\n'; - fs.writeFileSync(OUTPUT_PATH, output); - console.log(`Written ${rows.length} packages to ${OUTPUT_PATH}`); -}); + const output = `${lines.join("\n")}\n` + fs.writeFile(OUTPUT_PATH, output, (writeError) => { + if (writeError) { + throw writeError + } + process.stdout.write(`Written ${rows.length} packages to ${OUTPUT_PATH}\n`) + }) +}) diff --git a/src/actions/connection-action-creators.js b/src/actions/connection-action-creators.js index 5aa280a..4441c1f 100644 --- a/src/actions/connection-action-creators.js +++ b/src/actions/connection-action-creators.js @@ -52,8 +52,7 @@ import { setDatabaseStyles, fetchConfigurationDB, setThemeTint, - setCustomThemeEnabled, - SET_UI_THEME + setCustomThemeEnabled } from "actions/user-configurable-ui-action-creators" import { initializeAvailableBasemaps } from "charts/raster-chart/basemap" import { isServiceError } from "../services/util/is-service-error.util" diff --git a/src/charts/bubble/scatter.unit.test.js b/src/charts/bubble/scatter.unit.test.js index 21ebcec..4fc2fc0 100644 --- a/src/charts/bubble/scatter.unit.test.js +++ b/src/charts/bubble/scatter.unit.test.js @@ -96,8 +96,6 @@ const spec = { geoJson: null } -let setEliminateNull = false - const crossfilter = { dimension: () => ({ group: () => ({ @@ -106,10 +104,7 @@ const crossfilter = { }), binParams: noop }), - setEliminateNull: (val) => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - setEliminateNull = val - } + setEliminateNull: noop }) } diff --git a/src/charts/heat/heat.unit.test.js b/src/charts/heat/heat.unit.test.js index a213fe6..e23d256 100644 --- a/src/charts/heat/heat.unit.test.js +++ b/src/charts/heat/heat.unit.test.js @@ -130,8 +130,6 @@ const spec = { showColorPopup: false, rangeFilter: [] } -let setEliminateNull = false - const crossfilter = { dimension: () => ({ group: () => ({ @@ -141,10 +139,7 @@ const crossfilter = { }), binParams: noop }), - setEliminateNull: (val) => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - setEliminateNull = val - } + setEliminateNull: noop }) } diff --git a/src/charts/pie/pie.unit.test.js b/src/charts/pie/pie.unit.test.js index ecab23f..d3e438c 100644 --- a/src/charts/pie/pie.unit.test.js +++ b/src/charts/pie/pie.unit.test.js @@ -80,8 +80,6 @@ const defaultValues = { geoJson: null } -let setEliminateNull = false - const mockGroup = () => { const group = { reduce: () => group, @@ -100,10 +98,7 @@ const mockDimension = () => { filter: jest.fn(() => dimension), filterMulti: jest.fn(() => dimension), filterAll: jest.fn(() => dimension), - setEliminateNull: (val) => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - setEliminateNull = val - } + setEliminateNull: noop } return dimension } diff --git a/src/charts/row/row.unit.test.js b/src/charts/row/row.unit.test.js index 4388561..9b0ba73 100644 --- a/src/charts/row/row.unit.test.js +++ b/src/charts/row/row.unit.test.js @@ -70,8 +70,6 @@ const defaultValues = { geoJson: null } -let setEliminateNull = false - const crossfilter = { dimension: () => ({ group: () => ({ @@ -80,10 +78,7 @@ const crossfilter = { }), binParams: noop }), - setEliminateNull: (val) => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - setEliminateNull = val - } + setEliminateNull: noop }) } diff --git a/src/charts/table/table.unit.test.js b/src/charts/table/table.unit.test.js index d7b4b3d..e8f9f1c 100644 --- a/src/charts/table/table.unit.test.js +++ b/src/charts/table/table.unit.test.js @@ -43,8 +43,6 @@ const defaultValues = { geoJson: null } -let setEliminateNull = false - const crossfilter = { filter: noop, dimension: () => ({ @@ -55,10 +53,7 @@ const crossfilter = { }), binParams: noop }), - setEliminateNull: (val) => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - setEliminateNull = val - } + setEliminateNull: noop }) } diff --git a/src/charts/utils/event-listeners.unit.test.js b/src/charts/utils/event-listeners.unit.test.js index f3b0224..32c9924 100644 --- a/src/charts/utils/event-listeners.unit.test.js +++ b/src/charts/utils/event-listeners.unit.test.js @@ -20,9 +20,6 @@ describe("Chart Event Listeners", () => { let dcChart = null const id = 0 describe("Filtered Event Listener", () => { - const filters = [1, 2, 3] - const rangeFilter = ["date1", "date2"] - const areFiltersInverse = false /* beforeEach(() => { dispatch = jest.fn() @@ -71,13 +68,6 @@ describe("Chart Event Listeners", () => { }) describe("Sort Event Listener", () => { - const sortColumn = { - col: { - name: "val" - }, - index: 1, - order: "desc" - } /* beforeEach(() => { dispatch = jest.fn() diff --git a/src/components/account-panel/account-panel.js b/src/components/account-panel/account-panel.js index f34aa9d..2a2afe0 100644 --- a/src/components/account-panel/account-panel.js +++ b/src/components/account-panel/account-panel.js @@ -14,7 +14,7 @@ import { available_feature_flags } from "components/control-panel/featureflags" import IconAccount from "components/svg-icons/icon-account" -import { getThemes, currentTheme } from "utils/dark-mode-switcher" +import { getThemes } from "utils/dark-mode-switcher" import HelpMenu from "components/global-side-nav/HelpMenu" import { IMMERSE_UI_HELP_DROPDOWN, diff --git a/src/components/chart-type-panel/chart-type-panel.js b/src/components/chart-type-panel/chart-type-panel.js index 7c9a487..690abaa 100644 --- a/src/components/chart-type-panel/chart-type-panel.js +++ b/src/components/chart-type-panel/chart-type-panel.js @@ -143,8 +143,7 @@ export const filterChartTypes = ( distributed ) => { const enableDeckGLCharts = - !distributed && - (!isRenderingEnabled || getFeatureFlag(ENABLE_DECKGL_CHART)) + !distributed && (!isRenderingEnabled || getFeatureFlag(ENABLE_DECKGL_CHART)) const hiddenNonBEChartTypes = [ NOT_BE_RENDERED_CHART_TYPES.LINE, diff --git a/src/components/dashboards/dashboards.js b/src/components/dashboards/dashboards.js index b5dc6f8..d77bcdd 100644 --- a/src/components/dashboards/dashboards.js +++ b/src/components/dashboards/dashboards.js @@ -108,86 +108,86 @@ export default class Dashboards extends Component { return (
-
- {getFeatureFlag(LANDING_PAGE_PANELS) ? ( - { - storeInLocalStorage(WELCOME_PANEL_OPEN_KEY, open) - }} - > - - - ) : ( - "" - )} - - - -
- } - /> -
-
- +
+ {getFeatureFlag(LANDING_PAGE_PANELS) ? ( + { + storeInLocalStorage(WELCOME_PANEL_OPEN_KEY, open) + }} + > + + + ) : ( + "" + )} + + +
- {canCreateDashboard && !getFeatureFlag(GLOBAL_SIDE_NAV) && ( -
- - - - - {NEW_DASHBOARD_TEXT} - -
- )} + } + /> +
+
+
- + {canCreateDashboard && !getFeatureFlag(GLOBAL_SIDE_NAV) && ( +
+ + + + + {NEW_DASHBOARD_TEXT} + +
+ )}
+
+
) } } diff --git a/src/components/data-source-selector/utils.js b/src/components/data-source-selector/utils.js index def6192..deea445 100644 --- a/src/components/data-source-selector/utils.js +++ b/src/components/data-source-selector/utils.js @@ -90,4 +90,3 @@ export function processTablesListFromDashboardState( tables: allDataSources } } - diff --git a/src/components/login/login.tsx b/src/components/login/login.tsx index 88b6a29..e44d070 100644 --- a/src/components/login/login.tsx +++ b/src/components/login/login.tsx @@ -13,41 +13,44 @@ interface LoginProps { user: User dispatch: Function version: string + match: { + params?: { + database?: string + } + } } -class Login extends React.PureComponent { - render() { - const databaseParam = (this.props.match.params || {}).database - const login_panel_databases = this.props.user.login_panel_databases || [] +const Login = (props: LoginProps) => { + const databaseParam = (props.match.params || {}).database + const login_panel_databases = props.user.login_panel_databases || [] - const shouldDisableDatabase = - databaseParam !== undefined || this.props.shouldDisableDatabase + const shouldDisableDatabase = + databaseParam !== undefined || props.shouldDisableDatabase - if ( - databaseParam !== undefined && - login_panel_databases.length && - !login_panel_databases.some((v) => v === databaseParam) - ) { - return - } + if ( + databaseParam !== undefined && + login_panel_databases.length && + !login_panel_databases.some((v) => v === databaseParam) + ) { + return + } - return ( -
-
- -
+ return ( +
+
+
- ) - } +
+ ) } export default Login diff --git a/src/components/navigation-bar/navigation-bar-container.tsx b/src/components/navigation-bar/navigation-bar-container.tsx index 5d40b1a..c24ce8a 100644 --- a/src/components/navigation-bar/navigation-bar-container.tsx +++ b/src/components/navigation-bar/navigation-bar-container.tsx @@ -40,7 +40,6 @@ import { import { inDashboard, - atDashboardsList, atLoginPath, atLoggedOutPath, routeToSqlEditor, diff --git a/src/components/settings/settings-nav-item.tsx b/src/components/settings/settings-nav-item.tsx index fe1c250..c4f1dc2 100644 --- a/src/components/settings/settings-nav-item.tsx +++ b/src/components/settings/settings-nav-item.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import React, { useState, useContext } from "react" -import { useSelector, useDispatch } from "react-redux" +import { useDispatch } from "react-redux" import { NavLink, useHistory } from "react-router-dom" import { SimpleWarningDialog } from "widgets/dialog/Dialog" diff --git a/src/components/shared-settings/util.ts b/src/components/shared-settings/util.ts index 151b67b..725bf36 100644 --- a/src/components/shared-settings/util.ts +++ b/src/components/shared-settings/util.ts @@ -1,3 +1,2 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 - diff --git a/src/components/sql-notebook/data-preview/categorical/categorical-summary-chart.tsx b/src/components/sql-notebook/data-preview/categorical/categorical-summary-chart.tsx index f0c9e44..ed4cb95 100644 --- a/src/components/sql-notebook/data-preview/categorical/categorical-summary-chart.tsx +++ b/src/components/sql-notebook/data-preview/categorical/categorical-summary-chart.tsx @@ -62,7 +62,7 @@ export const CategoricalSummaryChart = ({ .finally(() => { setLoading(false) }) - }, [column, numBuckets]) + }, [column, numBuckets, theme]) return (
diff --git a/src/components/sql-notebook/data-preview/temporal/temporal-summary-chart.tsx b/src/components/sql-notebook/data-preview/temporal/temporal-summary-chart.tsx index a538929..5fb1ebd 100644 --- a/src/components/sql-notebook/data-preview/temporal/temporal-summary-chart.tsx +++ b/src/components/sql-notebook/data-preview/temporal/temporal-summary-chart.tsx @@ -72,7 +72,9 @@ export const TemporalSummaryChart = ({ .then((result: Array) => { const enrichedResult = result.map((row) => { const start = moment(row[bucketAlias]) - const end = start.clone().add(1, `${timeStep}s` as moment.unitOfTime.DurationConstructor) + const end = start + .clone() + .add(1, `${timeStep}s` as moment.unitOfTime.DurationConstructor) return { ...row, bucketRange: `${start.format(tooltipFormat)} – ${end.format( diff --git a/src/components/sql-notebook/left-panel/left-panel.tsx b/src/components/sql-notebook/left-panel/left-panel.tsx index d91857a..64ae764 100644 --- a/src/components/sql-notebook/left-panel/left-panel.tsx +++ b/src/components/sql-notebook/left-panel/left-panel.tsx @@ -21,9 +21,10 @@ export const LeftPanel = ({ const TABS = useMemo(() => { return [ - { label: "Table Browser", key: LeftPanelTabKey.DATA }, - { label: "Guidance", key: LeftPanelTabKey.GUIDANCE, enabled: iqEnabled } - ]}, [iqEnabled]) + { label: "Table Browser", key: LeftPanelTabKey.DATA }, + { label: "Guidance", key: LeftPanelTabKey.GUIDANCE, enabled: iqEnabled } + ] + }, [iqEnabled]) // eslint-disable-next-line init-declarations let activePanel @@ -44,15 +45,17 @@ export const LeftPanel = ({ return (
- {TABS.filter(({enabled}) => enabled !== false).map(({ label, key }) => ( - - ))} + {TABS.filter(({ enabled }) => enabled !== false).map( + ({ label, key }) => ( + + ) + )}
{activePanel}
diff --git a/src/components/sql-notebook/results/sql-notebook-answer-result.tsx b/src/components/sql-notebook/results/sql-notebook-answer-result.tsx index 151b67b..725bf36 100644 --- a/src/components/sql-notebook/results/sql-notebook-answer-result.tsx +++ b/src/components/sql-notebook/results/sql-notebook-answer-result.tsx @@ -1,3 +1,2 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 - diff --git a/src/components/svg-icons/index.js b/src/components/svg-icons/index.js index 88887d4..5509819 100644 --- a/src/components/svg-icons/index.js +++ b/src/components/svg-icons/index.js @@ -328,7 +328,6 @@ const SvgIcon = () => ( - ) diff --git a/src/reducers/user-configurable-ui-reducer.ts b/src/reducers/user-configurable-ui-reducer.ts index 3ef106f..2b7195d 100644 --- a/src/reducers/user-configurable-ui-reducer.ts +++ b/src/reducers/user-configurable-ui-reducer.ts @@ -30,7 +30,7 @@ import { getFeatureFlag, available_feature_flags } from "components/control-panel/featureflags" -import { DARK_THEME, ImmerseUITheme } from "utils/theme/types" +import { ImmerseUITheme } from "utils/theme/types" import { currentTheme } from "utils/dark-mode-switcher" const { DASHBOARD_GRID_MARGIN } = available_feature_flags diff --git a/src/vega/charts/combo-chart/combo-chart.ts b/src/vega/charts/combo-chart/combo-chart.ts index 446a679..d1c61a0 100644 --- a/src/vega/charts/combo-chart/combo-chart.ts +++ b/src/vega/charts/combo-chart/combo-chart.ts @@ -771,8 +771,7 @@ export function separateData( // rows as gaps so the path signal's datum.gap branch returns an empty // path, matching how we treat dimensions absent from the data entirely. const isNullMeasure = - isLineOrArea && - (datum.measure === null || datum.measure === undefined) + isLineOrArea && (datum.measure === null || datum.measure === undefined) // When connectNullsAcrossGaps is enabled, drop null-measure rows on // line/area entirely so Vega's lag/lead window transform on lineTable diff --git a/src/vega/charts/combo-chart/row-spec.ts b/src/vega/charts/combo-chart/row-spec.ts index 418de57..73efe58 100644 --- a/src/vega/charts/combo-chart/row-spec.ts +++ b/src/vega/charts/combo-chart/row-spec.ts @@ -1672,16 +1672,7 @@ export function buildSpec(opts: SpecOptions): Spec { type: "window", sort: { field: "sortableVal", order: "ascending" }, groupby: ["measureKey"], - ops: [ - "lag", - "lag", - "lag", - "lag", - "lead", - "lead", - "lead", - "lead" - ], + ops: ["lag", "lag", "lag", "lag", "lead", "lead", "lead", "lead"], fields: [ "hasValue", "measureMin", From 1830bdb62f75858e630dc5e700fc2b5c680b7ca4 Mon Sep 17 00:00:00 2001 From: Joe O'Hallaron Date: Mon, 28 Sep 2026 16:34:16 -0600 Subject: [PATCH 04/11] fix: disable puppeteer automation until BE sorted --- .github/workflows/immerse-puppeteer-automation.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/immerse-puppeteer-automation.yml b/.github/workflows/immerse-puppeteer-automation.yml index d7726f5..f01711e 100644 --- a/.github/workflows/immerse-puppeteer-automation.yml +++ b/.github/workflows/immerse-puppeteer-automation.yml @@ -1,13 +1,11 @@ name: immerse-puppeteer-automation +# Temporarily manual-only while isolated HeavyDB provisioning is designed. # Replaces the legacy Jenkins immerse-puppeteer-automation job. # The UI suite requires a HeavyDB instance containing the test data documented # in src/ui-tests/README-Puppeteer-UI-tests.md. on: - push: - branches: - - "**" workflow_dispatch: inputs: ref: From d70f8423b7c4a0e29c79453614c66a014a2765bf Mon Sep 17 00:00:00 2001 From: Joe O'Hallaron Date: Mon, 28 Sep 2026 16:52:09 -0600 Subject: [PATCH 05/11] fix: jest config for new package name --- jest.config.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jest.config.js b/jest.config.js index 90bfc0f..df67571 100644 --- a/jest.config.js +++ b/jest.config.js @@ -12,6 +12,8 @@ const baseConfig = { "^constants/(.*)$": "/src/constants/$1", "^services/(.*)$": "/src/services/$1", "^utils/(.*)$": "/src/utils/$1", + "^@heavyai/data-layer$": + "/node_modules/@heavyai/data-layer/packages/data-layer/dist/heavyai-data-layer.js", "^vega-lite$": "/node_modules/vega-lite/build/vega-lite.min.js", "^fs$": "/test-config/__mocks__/fs.js", @@ -23,7 +25,7 @@ const baseConfig = { "\\.[jt]sx?$": "babel-jest" }, transformIgnorePatterns: [ - "/node_modules/(?!(@heavyai|robust-predicates|delaunator|d3-.*|axios|internmap|formdata-polyfill|fetch-blob|data-uri-to-buffer|node-fetch|vega|suneditor|legendables|@mapbox-controls|@mapbox|vega-.*|cheerio|cheerio-.*))" + "/node_modules/(?!(@heavyai|robust-predicates|delaunator|d3-.*|axios|internmap|formdata-polyfill|fetch-blob|data-uri-to-buffer|node-fetch|vega|suneditor|quill-next|@quill-next|parchment|lodash-es|legendables|@mapbox-controls|@mapbox|vega-.*|cheerio|cheerio-.*))" ], setupFiles: ["jest-canvas-mock", "/test-config/jest-shim"] } From 8ac8703e08063d17759fd4adec39a43a0527e942 Mon Sep 17 00:00:00 2001 From: Joe O'Hallaron Date: Mon, 28 Sep 2026 22:15:05 -0600 Subject: [PATCH 06/11] feat: run puppeteer against disposable heavydb --- .../immerse-puppeteer-automation.yml | 85 +++-- scripts/ci/puppeteer-heavydb/Dockerfile | 37 +++ .../ci/puppeteer-heavydb/cleanup-heavydb.sh | 21 ++ .../puppeteer-heavydb/collect-diagnostics.sh | 24 ++ .../download-heavydb-artifact.sh | 156 +++++++++ scripts/ci/puppeteer-heavydb/entrypoint.sh | 52 +++ scripts/ci/puppeteer-heavydb/load-fixtures.sh | 43 +++ scripts/ci/puppeteer-heavydb/start-heavydb.sh | 64 ++++ ...board-action-creators-basemap.unit.test.js | 32 ++ src/actions/dashboard-action-creators.ts | 2 +- .../data-source-selector-utils.unit.test.js | 99 ++---- src/components/logo/logo.unit.test.js | 7 +- src/ui-tests/README-Puppeteer-UI-tests.md | 31 +- src/ui-tests/chart-coloring.ui.test.ts | 74 ++--- .../chart-creation/table-chart.ui.test.ts | 14 +- .../vega-combo/vega-combo-creation.ui.test.ts | 10 +- .../fixtures/heavydb/generate-fixtures.js | 303 ++++++++++++++++++ src/ui-tests/fixtures/heavydb/schema.sql | 87 +++++ src/ui-tests/utils/dashboards.ts | 21 ++ 19 files changed, 999 insertions(+), 163 deletions(-) create mode 100644 scripts/ci/puppeteer-heavydb/Dockerfile create mode 100755 scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh create mode 100755 scripts/ci/puppeteer-heavydb/collect-diagnostics.sh create mode 100755 scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh create mode 100755 scripts/ci/puppeteer-heavydb/entrypoint.sh create mode 100755 scripts/ci/puppeteer-heavydb/load-fixtures.sh create mode 100755 scripts/ci/puppeteer-heavydb/start-heavydb.sh create mode 100755 src/ui-tests/fixtures/heavydb/generate-fixtures.js create mode 100644 src/ui-tests/fixtures/heavydb/schema.sql diff --git a/.github/workflows/immerse-puppeteer-automation.yml b/.github/workflows/immerse-puppeteer-automation.yml index f01711e..274759a 100644 --- a/.github/workflows/immerse-puppeteer-automation.yml +++ b/.github/workflows/immerse-puppeteer-automation.yml @@ -1,9 +1,7 @@ name: immerse-puppeteer-automation -# Temporarily manual-only while isolated HeavyDB provisioning is designed. +# Temporarily manual-only until the disposable HeavyDB path has a successful run. # Replaces the legacy Jenkins immerse-puppeteer-automation job. -# The UI suite requires a HeavyDB instance containing the test data documented -# in src/ui-tests/README-Puppeteer-UI-tests.md. on: workflow_dispatch: @@ -12,6 +10,10 @@ on: description: "Branch or SHA to test" type: string default: "" + heavydb_run_id: + description: "Product Builder run ID (leave empty for latest published master build)" + type: string + default: "" permissions: contents: read @@ -24,7 +26,11 @@ jobs: immerse-puppeteer-automation: name: immerse-puppeteer-automation runs-on: linux-amd64-cpu32 - timeout-minutes: 90 + timeout-minutes: 120 + env: + HEAVYDB_RESOURCE_SUFFIX: ${{ github.run_id }}-${{ github.run_attempt }} + HEAVYDB_PRODUCT_DIR: ${{ runner.temp }}/heavydb-product-${{ github.run_id }}-${{ github.run_attempt }} + HEAVYDB_DIAGNOSTICS_DIR: ${{ github.workspace }}/heavydb-diagnostics steps: - name: Checkout uses: actions/checkout@v4 @@ -54,33 +60,45 @@ jobs: - name: Install dependencies run: npm ci - - name: Configure test server + - name: Check artifact download prerequisites + run: | + git lfs version + jq --version + unzip -v + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Download and verify HeavyDB product + id: heavydb-product env: - TEST_DATABASE: ${{ secrets.PLAYWRIGHT_TEST_DB }} - TEST_USERNAME: ${{ secrets.PLAYWRIGHT_TEST_USER }} - TEST_PASSWORD: ${{ secrets.PLAYWRIGHT_TEST_PASSWORD }} - TEST_SERVER: ${{ secrets.PLAYWRIGHT_TEST_SERVER }} + HEAVYAI_BINARY_SCAN_TOKEN: ${{ secrets.HEAVYAI_BINARY_SCAN_TOKEN }} + HEAVYDB_RUN_ID: ${{ inputs.heavydb_run_id }} run: | - node <<'NODE' - const fs = require("fs") + args=(--output-dir "$HEAVYDB_PRODUCT_DIR") + if [ -n "$HEAVYDB_RUN_ID" ]; then + args+=(--run-id "$HEAVYDB_RUN_ID") + fi + scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh "${args[@]}" - const required = [ - "TEST_DATABASE", - "TEST_USERNAME", - "TEST_PASSWORD", - "TEST_SERVER" - ] - const missing = required.filter((name) => !process.env[name]) - if (missing.length) { - throw new Error(`Missing required secrets: ${missing.join(", ")}`) - } + - name: Start disposable HeavyDB + env: + HEAVYDB_TARBALL: ${{ steps.heavydb-product.outputs.tarball }} + run: scripts/ci/puppeteer-heavydb/start-heavydb.sh + + - name: Load deterministic test fixtures + run: scripts/ci/puppeteer-heavydb/load-fixtures.sh + - name: Configure test server + run: | + node <<'NODE' + const fs = require("fs") const server = { - database: process.env.TEST_DATABASE, + database: "heavyai", master: true, - username: process.env.TEST_USERNAME, - password: process.env.TEST_PASSWORD, - url: process.env.TEST_SERVER, + username: "admin", + password: "HyperInteractive", + url: "http://127.0.0.1:6273/", GTM: "GTM-MDD888", theme: "dark" } @@ -119,14 +137,29 @@ jobs: kill "$WEBPACK_SERVER_PID" 2>/dev/null || true fi + - name: Collect HeavyDB diagnostics + if: always() + run: | + if [ -x scripts/ci/puppeteer-heavydb/collect-diagnostics.sh ]; then + scripts/ci/puppeteer-heavydb/collect-diagnostics.sh + fi + - name: Upload test diagnostics if: always() uses: actions/upload-artifact@v4 with: - name: puppeteer-test-diagnostics + name: puppeteer-test-diagnostics-${{ github.run_attempt }} path: | + heavydb-diagnostics jest-results.xml webpack.log src/ui-tests/_screenshot-*.png if-no-files-found: ignore retention-days: 14 + + - name: Remove disposable HeavyDB resources + if: always() + run: | + if [ -x scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh ]; then + scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh + fi diff --git a/scripts/ci/puppeteer-heavydb/Dockerfile b/scripts/ci/puppeteer-heavydb/Dockerfile new file mode 100644 index 0000000..b68ae53 --- /dev/null +++ b/scripts/ci/puppeteer-heavydb/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:22.04 AS extract + +WORKDIR /opt/heavyai +COPY product.tar.gz /tmp/product.tar.gz +RUN tar xzf /tmp/product.tar.gz --strip-components=1 -C /opt/heavyai + +FROM nvcr.io/nvidia/cuda:12.9.1-runtime-ubuntu22.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + bsdmainutils \ + ca-certificates \ + clang-14 \ + curl \ + default-jre-headless \ + libegl1 \ + libglx0 \ + liblz4-tool \ + libopengl0 \ + libpython3.10 \ + procps \ + python3.10 \ + python3.10-venv \ + zip \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=extract /opt/heavyai /opt/heavyai +COPY entrypoint.sh /opt/heavyai/ci-entrypoint.sh +RUN chmod +x /opt/heavyai/ci-entrypoint.sh + +WORKDIR /opt/heavyai +EXPOSE 6273 6274 6278 6279 + +HEALTHCHECK --interval=5s --timeout=5s --start-period=20s --retries=24 \ + CMD printf 'SELECT 1;\n' | /opt/heavyai/bin/heavysql heavyai -u admin -p HyperInteractive >/dev/null + +ENTRYPOINT ["/opt/heavyai/ci-entrypoint.sh"] diff --git a/scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh b/scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh new file mode 100755 index 0000000..28da87c --- /dev/null +++ b/scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +: "${HEAVYDB_RESOURCE_SUFFIX:?HEAVYDB_RESOURCE_SUFFIX is required}" + +resource_suffix="$(tr -cd '[:alnum:]_-' <<<"${HEAVYDB_RESOURCE_SUFFIX}")" +readonly container_name="immerse-puppeteer-heavydb-${resource_suffix}" +readonly image_name="immerse-puppeteer-heavydb:${resource_suffix}" +readonly volume_name="immerse-puppeteer-heavydb-${resource_suffix}" +readonly build_context="${RUNNER_TEMP:-/tmp}/immerse-heavydb-build-${resource_suffix}" + +docker rm --force "${container_name}" 2>/dev/null || true +docker volume rm --force "${volume_name}" 2>/dev/null || true +docker image rm --force "${image_name}" 2>/dev/null || true +rm -rf "${build_context}" + +if [[ -n "${HEAVYDB_PRODUCT_DIR:-}" ]]; then + rm -rf "${HEAVYDB_PRODUCT_DIR}" +fi diff --git a/scripts/ci/puppeteer-heavydb/collect-diagnostics.sh b/scripts/ci/puppeteer-heavydb/collect-diagnostics.sh new file mode 100755 index 0000000..97b5c16 --- /dev/null +++ b/scripts/ci/puppeteer-heavydb/collect-diagnostics.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +: "${HEAVYDB_RESOURCE_SUFFIX:?HEAVYDB_RESOURCE_SUFFIX is required}" +: "${HEAVYDB_DIAGNOSTICS_DIR:?HEAVYDB_DIAGNOSTICS_DIR is required}" + +resource_suffix="$(tr -cd '[:alnum:]_-' <<<"${HEAVYDB_RESOURCE_SUFFIX}")" +readonly container_name="immerse-puppeteer-heavydb-${resource_suffix}" + +mkdir -p "${HEAVYDB_DIAGNOSTICS_DIR}" +if ! docker inspect "${container_name}" >"${HEAVYDB_DIAGNOSTICS_DIR}/container-inspect.json" 2>&1; then + echo "Container ${container_name} does not exist; no HeavyDB diagnostics to collect" + exit 0 +fi + +docker logs "${container_name}" >"${HEAVYDB_DIAGNOSTICS_DIR}/container.log" 2>&1 || true +docker cp \ + "${container_name}:/var/lib/heavyai/ci-logs" \ + "${HEAVYDB_DIAGNOSTICS_DIR}/ci-logs" 2>/dev/null || true +docker cp \ + "${container_name}:/var/lib/heavyai/storage/log" \ + "${HEAVYDB_DIAGNOSTICS_DIR}/heavydb-logs" 2>/dev/null || true diff --git a/scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh b/scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh new file mode 100755 index 0000000..46de6ce --- /dev/null +++ b/scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +readonly HEAVYDB_REPOSITORY="heavyai/heavydb" +readonly PRODUCT_WORKFLOW="rc-builder.yml" +readonly SCAN_REPOSITORY="heavyai/heavyai-binary-scan-testing" +readonly ARTIFACT_PATTERN='^heavyai-.*-ubuntu22\.04-x86_64-render-signed\.zip$' +readonly MINIMUM_ARTIFACT_BYTES=100000000 + +output_dir="" +requested_run_id="" + +while (($#)); do + case "$1" in + --output-dir) + output_dir="$2" + shift 2 + ;; + --run-id) + requested_run_id="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + exit 2 + ;; + esac +done + +: "${output_dir:?--output-dir is required}" +: "${HEAVYAI_BINARY_SCAN_TOKEN:?HEAVYAI_BINARY_SCAN_TOKEN is required}" + +api() { + curl --fail --silent --show-error \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${HEAVYAI_BINARY_SCAN_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/$1" +} + +find_artifact_for_run() { + local run_id="$1" + local run_json run_date directory_json artifact_name + + run_json="$(api "repos/${HEAVYDB_REPOSITORY}/actions/runs/${run_id}")" + if [[ "$(jq -r '.path' <<<"${run_json}")" != ".github/workflows/${PRODUCT_WORKFLOW}" ]]; then + echo "Run ${run_id} is not a Product Builder run" >&2 + return 1 + fi + if [[ "$(jq -r '.head_branch' <<<"${run_json}")" != "master" ]] || + [[ "$(jq -r '.conclusion' <<<"${run_json}")" != "success" ]]; then + echo "Run ${run_id} is not a successful master build" >&2 + return 1 + fi + + run_date="$(jq -r '.created_at[0:10]' <<<"${run_json}")" + if ! directory_json="$(api "repos/${SCAN_REPOSITORY}/contents/runs/${run_date}/${run_id}" 2>/dev/null)"; then + return 1 + fi + + artifact_name="$( + jq -r '.[].name' <<<"${directory_json}" | + awk -v pattern="${ARTIFACT_PATTERN}" '$0 ~ pattern { print; exit }' + )" + [[ -n "${artifact_name}" ]] || return 1 + + printf '%s\t%s\t%s\n' "${run_id}" "${run_date}" "${artifact_name}" +} + +resolve_artifact() { + local candidate + + if [[ -n "${requested_run_id}" ]]; then + find_artifact_for_run "${requested_run_id}" + return + fi + + while IFS= read -r candidate; do + if find_artifact_for_run "${candidate}"; then + return + fi + echo "Product Builder run ${candidate} has no published Ubuntu x86_64 bundle; trying the previous run" >&2 + done < <( + api "repos/${HEAVYDB_REPOSITORY}/actions/workflows/${PRODUCT_WORKFLOW}/runs?branch=master&status=success&per_page=100" | + jq -r '.workflow_runs[].id' + ) + + echo "No successful Product Builder run with a published Ubuntu x86_64 bundle was found" >&2 + return 1 +} + +IFS=$'\t' read -r run_id run_date artifact_name < <(resolve_artifact) +artifact_relative_path="runs/${run_date}/${run_id}/${artifact_name}" + +mkdir -p "${output_dir}" +output_dir="$(cd "${output_dir}" && pwd)" +checkout_dir="${output_dir}/scan-repository" +bundle_dir="${output_dir}/bundle" + +cleanup_credentials() { + if [[ -d "${checkout_dir}/.git" ]]; then + git -C "${checkout_dir}" remote set-url origin "https://github.com/${SCAN_REPOSITORY}.git" || true + fi +} +trap cleanup_credentials EXIT + +rm -rf "${checkout_dir}" "${bundle_dir}" +GIT_LFS_SKIP_SMUDGE=1 git clone --filter=blob:none --no-checkout \ + "https://x-access-token:${HEAVYAI_BINARY_SCAN_TOKEN}@github.com/${SCAN_REPOSITORY}.git" \ + "${checkout_dir}" +git -C "${checkout_dir}" sparse-checkout init --cone +git -C "${checkout_dir}" sparse-checkout set "runs/${run_date}/${run_id}" +git -C "${checkout_dir}" checkout +git -C "${checkout_dir}" lfs pull --include="${artifact_relative_path}" --exclude="" +cleanup_credentials + +artifact_path="${checkout_dir}/${artifact_relative_path}" +artifact_size="$(wc -c <"${artifact_path}" | tr -d ' ')" +if ((artifact_size < MINIMUM_ARTIFACT_BYTES)); then + echo "Downloaded artifact is only ${artifact_size} bytes; Git LFS content was not retrieved" >&2 + exit 1 +fi + +mkdir -p "${bundle_dir}" +unzip -q "${artifact_path}" -d "${bundle_dir}" +tarball_path="$( + find "${bundle_dir}" -maxdepth 1 -type f -name '*.tar.gz' -print -quit +)" +if [[ -z "${tarball_path}" ]]; then + echo "Signed bundle does not contain a product tarball" >&2 + exit 1 +fi +bundle_path="${tarball_path}.bundle" +if [[ ! -f "${bundle_path}" ]]; then + echo "Signed bundle does not contain $(basename "${bundle_path}")" >&2 + exit 1 +fi + +cosign verify-blob \ + --bundle="${bundle_path}" \ + --certificate-identity-regexp='https://github.com/heavyai/heavydb/.github/workflows/rc-builder.yml@refs/heads/.*' \ + --certificate-oidc-issuer='https://token.actions.githubusercontent.com' \ + "${tarball_path}" + +rm -rf "${checkout_dir}" + +echo "Verified HeavyDB product from run ${run_id}: $(basename "${tarball_path}")" +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + { + echo "run_id=${run_id}" + echo "artifact_name=${artifact_name}" + echo "tarball=${tarball_path}" + } >>"${GITHUB_OUTPUT}" +fi diff --git a/scripts/ci/puppeteer-heavydb/entrypoint.sh b/scripts/ci/puppeteer-heavydb/entrypoint.sh new file mode 100755 index 0000000..b6a23b9 --- /dev/null +++ b/scripts/ci/puppeteer-heavydb/entrypoint.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +readonly data_dir="${HEAVYDB_DATA_DIR:-/var/lib/heavyai/storage}" +readonly console_log_dir="/var/lib/heavyai/ci-logs" + +mkdir -p "${console_log_dir}" +if [[ ! -d "${data_dir}/mapd_catalogs" ]]; then + mkdir -p "${data_dir}" + /opt/heavyai/bin/initheavy -f --data "${data_dir}" +fi + +children=() +stop_services() { + trap - SIGINT SIGTERM EXIT + if ((${#children[@]})); then + kill "${children[@]}" 2>/dev/null || true + wait "${children[@]}" 2>/dev/null || true + fi +} +trap stop_services SIGINT SIGTERM EXIT + +/opt/heavyai/bin/heavydb "${data_dir}" \ + --cpu-only \ + --port 6274 \ + --http-port 6278 \ + --calcite-port 6279 \ + >"${console_log_dir}/heavydb-console.log" 2>&1 & +children+=("$!") +echo "HeavyDB started as PID ${children[-1]}" + +/opt/heavyai/bin/heavy_web_server \ + --port 6273 \ + --backend-url http://127.0.0.1:6278 \ + --data "${data_dir}" \ + >"${console_log_dir}/heavy-web-server-console.log" 2>&1 & +children+=("$!") +echo "Heavy web server started as PID ${children[-1]}" + +set +e +wait -n "${children[@]}" +status=$? +set -e + +echo "A HeavyDB service exited with status ${status}" >&2 +for log_file in "${console_log_dir}"/*.log; do + echo "===== ${log_file} =====" >&2 + cat "${log_file}" >&2 +done +exit "${status}" diff --git a/scripts/ci/puppeteer-heavydb/load-fixtures.sh b/scripts/ci/puppeteer-heavydb/load-fixtures.sh new file mode 100755 index 0000000..53f10c8 --- /dev/null +++ b/scripts/ci/puppeteer-heavydb/load-fixtures.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +: "${HEAVYDB_RESOURCE_SUFFIX:?HEAVYDB_RESOURCE_SUFFIX is required}" + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repository_root="$(cd "${script_dir}/../../.." && pwd)" +resource_suffix="$(tr -cd '[:alnum:]_-' <<<"${HEAVYDB_RESOURCE_SUFFIX}")" +readonly container_name="immerse-puppeteer-heavydb-${resource_suffix}" +readonly generated_dir="${RUNNER_TEMP:-/tmp}/immerse-heavydb-fixtures-${resource_suffix}" +readonly container_fixture_dir="/var/lib/heavyai/storage/import/immerse-ci" +readonly heavysql="/opt/heavyai/bin/heavysql heavyai -u admin -p HyperInteractive" + +rm -rf "${generated_dir}" +node \ + "${repository_root}/src/ui-tests/fixtures/heavydb/generate-fixtures.js" \ + "${generated_dir}" + +docker exec "${container_name}" mkdir -p "${container_fixture_dir}" +docker cp "${generated_dir}/." "${container_name}:${container_fixture_dir}" +docker exec -i "${container_name}" bash -c "${heavysql}" < \ + "${repository_root}/src/ui-tests/fixtures/heavydb/schema.sql" + +validation="$( + docker exec "${container_name}" bash -c "${heavysql}" <<'SQL' +SELECT CASE WHEN + (SELECT COUNT(*) FROM flights_donotmodify) = 1200 + AND (SELECT COUNT(*) FROM flights_donotmodify WHERE flight_month = 100) = 0 + AND (SELECT COUNT(*) FROM flights_donotmodify WHERE flight_month IN (5, 12)) > 0 + AND (SELECT COUNT(DISTINCT country) FROM tweets_nov_feb) = 51 + AND (SELECT COUNT(*) FROM us_states_geo) = 5 +THEN 'PASS' ELSE 'FAIL' END AS fixture_status; +SQL +)" +echo "${validation}" +if [[ "${validation}" != *PASS* ]]; then + echo "HeavyDB fixture validation failed" >&2 + exit 1 +fi + +rm -rf "${generated_dir}" diff --git a/scripts/ci/puppeteer-heavydb/start-heavydb.sh b/scripts/ci/puppeteer-heavydb/start-heavydb.sh new file mode 100755 index 0000000..4894aa8 --- /dev/null +++ b/scripts/ci/puppeteer-heavydb/start-heavydb.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +: "${HEAVYDB_TARBALL:?HEAVYDB_TARBALL is required}" +: "${HEAVYDB_RESOURCE_SUFFIX:?HEAVYDB_RESOURCE_SUFFIX is required}" + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +resource_suffix="$(tr -cd '[:alnum:]_-' <<<"${HEAVYDB_RESOURCE_SUFFIX}")" +readonly container_name="immerse-puppeteer-heavydb-${resource_suffix}" +readonly image_name="immerse-puppeteer-heavydb:${resource_suffix}" +readonly volume_name="immerse-puppeteer-heavydb-${resource_suffix}" +readonly build_context="${RUNNER_TEMP:-/tmp}/immerse-heavydb-build-${resource_suffix}" + +rm -rf "${build_context}" +mkdir -p "${build_context}" +cp "${HEAVYDB_TARBALL}" "${build_context}/product.tar.gz" +cp "${script_dir}/Dockerfile" "${script_dir}/entrypoint.sh" "${build_context}/" + +docker build --tag "${image_name}" "${build_context}" +docker volume create "${volume_name}" >/dev/null +docker run --detach \ + --name "${container_name}" \ + --volume "${volume_name}:/var/lib/heavyai" \ + --publish 127.0.0.1:6273:6273 \ + --publish 127.0.0.1:6274:6274 \ + --publish 127.0.0.1:6278:6278 \ + --publish 127.0.0.1:6279:6279 \ + "${image_name}" >/dev/null + +ready=false +for _ in $(seq 1 90); do + if ! docker inspect "${container_name}" --format '{{.State.Running}}' | grep -q true; then + echo "HeavyDB container exited before becoming ready" >&2 + docker logs "${container_name}" >&2 || true + exit 1 + fi + + if docker exec "${container_name}" bash -c \ + "printf 'SELECT 1;\\n' | /opt/heavyai/bin/heavysql heavyai -u admin -p HyperInteractive >/dev/null" && + curl --fail --silent --show-error http://127.0.0.1:6273/ >/dev/null; then + ready=true + break + fi + sleep 2 +done + +if [[ "${ready}" != "true" ]]; then + echo "HeavyDB did not become ready within 180 seconds" >&2 + docker logs "${container_name}" >&2 || true + exit 1 +fi + +rm -rf "${build_context}" +echo "Disposable HeavyDB is ready at http://127.0.0.1:6273" + +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + { + echo "container=${container_name}" + echo "image=${image_name}" + echo "volume=${volume_name}" + } >>"${GITHUB_OUTPUT}" +fi diff --git a/src/actions/dashboard-action-creators-basemap.unit.test.js b/src/actions/dashboard-action-creators-basemap.unit.test.js index fa8d54a..4263e4d 100644 --- a/src/actions/dashboard-action-creators-basemap.unit.test.js +++ b/src/actions/dashboard-action-creators-basemap.unit.test.js @@ -120,4 +120,36 @@ describe("dashboard-action-creators minimalist basemap", () => { "minimalist" ) }) + + it("updates imported dashboard titles inside tabs", async () => { + const services = new Map() + const createDashboardAsync = jest.fn().mockResolvedValue(undefined) + services.set("DbCon", { + getDashboardsAsync: jest.fn().mockResolvedValue([]), + createDashboardAsync + }) + + const importState = JSON.stringify({ + tabs: { + tab1: { + dashboard: { + title: "Exported title" + } + } + } + }) + + await importDashboard("Imported title", "{}", importState)( + jest.fn(), + () => ({}), + services + ) + + const serializedImportedState = createDashboardAsync.mock.calls[0][1] + const decodedImportedState = decodeSerializedState(serializedImportedState) + + expect(decodedImportedState.tabs.tab1.dashboard.title).toBe( + "Imported title" + ) + }) }) diff --git a/src/actions/dashboard-action-creators.ts b/src/actions/dashboard-action-creators.ts index 6fe2fc6..55c593e 100644 --- a/src/actions/dashboard-action-creators.ts +++ b/src/actions/dashboard-action-creators.ts @@ -894,7 +894,7 @@ export const importDashboard = (title, metadata, state) => async ( const deserializedState = normalizeMinimalistBasemapState(JSON.parse(state)) const newTitleState = dashboardStateWithNewTitle( - state.tabs, + deserializedState.tabs, deserializedState, uniqueTitle ) diff --git a/src/components/data-source-selector/data-source-selector-utils.unit.test.js b/src/components/data-source-selector/data-source-selector-utils.unit.test.js index 8e12580..5b2e692 100644 --- a/src/components/data-source-selector/data-source-selector-utils.unit.test.js +++ b/src/components/data-source-selector/data-source-selector-utils.unit.test.js @@ -1,77 +1,36 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { renderHook } from "@testing-library/react-hooks" -import { Provider } from "react-redux" -import React from "react" -import { useShouldShowEnterprisePrompt } from "./utils" -import thunk from "redux-thunk" -import configureStore from "redux-mock-store" - -const mockState = { - dashboard: { - dataSources: {}, - loadState: {} - }, - connection: { - isMSDEnabled: false - }, - dc: { - render: { - error: false - }, - redraw: { - error: false - } - }, - chartEditor: { - savedDataSources: { dataSources: { tweets: true } }, - savedCharts: { - 1: {} - }, - savedFilters: { - 1: [] - } - }, - charts: { - 1: {}, - 2: {}, - 3: {} - }, - ui: { - selectorPillHover: { shouldShowPrompt: false, top: false } - }, - tables: { list: [] } -} - -describe("useShouldShowEnterprisePrompt test suite", () => { - /* eslint-disable react/display-name */ - const middlewares = [thunk] - const mockStore = configureStore(middlewares) - it("can useShouldShowEnterprisePrompt when multisource is disabled", () => { - const store = mockStore(mockState) - const { result } = renderHook(() => useShouldShowEnterprisePrompt(), { - wrapper: ({ children }) => {children} - }) - - const res = result.current("flights") - - expect(res).toEqual(true) +import { processTablesListFromDashboardState } from "./utils" + +describe("processTablesListFromDashboardState", () => { + it("keeps current sources first and sorts unused tables", () => { + const result = processTablesListFromDashboardState({ flights: true }, [ + { name: "zebra" }, + { name: "flights" }, + { name: "alpha" } + ]) + + expect(result.dataSources).toEqual({ flights: true }) + expect(result.tables.map(({ label }) => label)).toEqual([ + "flights", + "", + "alpha", + "zebra" + ]) }) - it("can useShouldShowEnterprisePrompt when multisource is enabled", () => { - const store = mockStore({ - ...mockState, - connection: { - isMSDEnabled: true - } - }) - const { result } = renderHook(() => useShouldShowEnterprisePrompt(), { - wrapper: ({ children }) => {children} - }) - - const res = result.current("flights") - - expect(res).toEqual(false) + it("uses a table dataSource value when one is provided", () => { + const result = processTablesListFromDashboardState( + { "parameter.source": true }, + [{ name: "Selected table", dataSource: "parameter.source" }] + ) + + expect(result.tables[0]).toEqual( + expect.objectContaining({ + label: "Selected table", + value: "parameter.source" + }) + ) }) }) diff --git a/src/components/logo/logo.unit.test.js b/src/components/logo/logo.unit.test.js index 1c1b302..6b9d554 100644 --- a/src/components/logo/logo.unit.test.js +++ b/src/components/logo/logo.unit.test.js @@ -42,7 +42,7 @@ describe("Logo", () => { it("should mount logo after config loaded", () => { state.connection.serversJsonPending = false - const { getByText, queryByRole } = render( + const { container, getByText, getByRole } = render( @@ -50,8 +50,9 @@ describe("Logo", () => { // Check that the text logo exists expect(getByText("HeavyAI")).toBeInTheDocument() - // Check that no image is rendered - expect(queryByRole("img")).not.toBeInTheDocument() + // Check that the NVIDIA logo is rendered alongside it + expect(getByRole("img")).toBeInTheDocument() + expect(container.querySelector("img.nv-logo")).toBeInTheDocument() }) it("should mount an image if a logoURL is provided", () => { diff --git a/src/ui-tests/README-Puppeteer-UI-tests.md b/src/ui-tests/README-Puppeteer-UI-tests.md index ad45629..fa74098 100644 --- a/src/ui-tests/README-Puppeteer-UI-tests.md +++ b/src/ui-tests/README-Puppeteer-UI-tests.md @@ -2,17 +2,28 @@ ## Configuration requirements -The UI tests require connecting to a server running a HeavyDB instance with specific data sources: - -- flights_donotmodify -- us_states_geo -- tweets_nov_feb - -This list of required data sources must be updated if new data sources are incorporated into UI tests. +CI starts a disposable HeavyDB and loads deterministic CSV fixtures generated +by `fixtures/heavydb/generate-fixtures.js`. Local runs may use the same loader +or a server containing the following contract: + +- `flights_donotmodify` has 1,200 synthetic flight rows. It contains the + standard flight columns used throughout the tests, including carrier, + destination, origin, delay, date, location, and plane metadata. Months 5 and + 12 are present, month 100 is absent, and carrier values include American + Airlines, Delta Air Lines, Southwest Airlines, and United Airlines. +- `tweets_nov_feb` has deterministic coordinate clusters and all 51 country + codes asserted by the scatter and point-map tests. It also includes + `followees`, `followers`, `state_abbr`, `admin1`, and `join_time`. +- `us_states_geo` has five state rows with `NAME`, `ALAND`, and `AWATER` + columns. The active tests use it for SQL view creation and deletion. + +The schema is defined in `fixtures/heavydb/schema.sql`. Update the generator, +schema, validation in `scripts/ci/puppeteer-heavydb/load-fixtures.sh`, and this +contract together when a UI test requires another table or column. ## How to run in the background -In your `servers.local.json`, specify a server with the required data sources (e.g. Forge). +In your `servers.local.json`, specify a server with the required data sources. Start a local server hosting Immerse at `http://localhost:8002`. @@ -20,7 +31,7 @@ Run tests: `npm run test:ui` ## How to watch it run -In your `servers.local.json`, specify a server with the required data sources (e.g. Forge). +In your `servers.local.json`, specify a server with the required data sources. Start a local server hosting Immerse at `http://localhost:8002`. @@ -35,7 +46,7 @@ describe("DEBUG A suite of tests", () => { it("DEBUG A single test", async () => { ``` -In your `servers.local.json`, specify a server with the required data sources (e.g. Forge). +In your `servers.local.json`, specify a server with the required data sources. Start a local server hosting Immerse at `http://localhost:8002`. diff --git a/src/ui-tests/chart-coloring.ui.test.ts b/src/ui-tests/chart-coloring.ui.test.ts index e1a3ad2..523caaa 100644 --- a/src/ui-tests/chart-coloring.ui.test.ts +++ b/src/ui-tests/chart-coloring.ui.test.ts @@ -6,11 +6,15 @@ import { createChart, createDashboard, deleteSelector, - expectAttribute, waitForVisible, processPause } from "./utils" +const getPieColors = () => + page.$$eval(".pie-slice > path", (slices) => + slices.map((slice) => slice.getAttribute("fill")) + ) + describe("Chart coloring", () => { beforeAll(async () => { await createDashboard("ui-test/chart-coloring") @@ -24,18 +28,15 @@ describe("Chart coloring", () => { it("Adding and Removing Color Measure", async () => { await waitForVisible(".pie-slice") - await expectAttribute(".pie-slice._0 > path", "fill", "#d0f400") - await expectAttribute(".pie-slice._3 > path", "fill", "#48b5c4") - await expectAttribute(".pie-slice._6 > path", "fill", "#1984c5") - await expectAttribute(".pie-slice._9 > path", "fill", "#115f9a") + const colorsWithMeasure = await getPieColors() + expect(colorsWithMeasure.length).toBeGreaterThan(3) await deleteSelector("measures", 2) await processPause() - await expectAttribute(".pie-slice._0 > path", "fill", "#bdcf32") - await expectAttribute(".pie-slice._3 > path", "fill", "#ef9b20") - await expectAttribute(".pie-slice._6 > path", "fill", "#27aeef") - await expectAttribute(".pie-slice._9 > path", "fill", "#ede15b") + const colorsWithoutMeasure = await getPieColors() + expect(colorsWithoutMeasure).not.toEqual(colorsWithMeasure) + expect(new Set(colorsWithoutMeasure).size).toBeGreaterThan(1) }) it("Adding Solid Colors", async () => { @@ -45,47 +46,24 @@ describe("Chart coloring", () => { await clickAfterVisible(".swatch-group.solid > div:nth-child(2)") await processPause() - await expectAttribute(".pie-slice._0 > path", "fill", "#f46a9b") - await expectAttribute(".pie-slice._3 > path", "fill", "#f46a9b") - await expectAttribute(".pie-slice._6 > path", "fill", "#f46a9b") - await expectAttribute(".pie-slice._9 > path", "fill", "#f46a9b") + const solidColors = await getPieColors() + expect(new Set(solidColors)).toEqual(new Set(["#f46a9b"])) }) it("Adding Ordinal Colors", async () => { - await clickAfterVisible(".color-swatch.selected") - await clickAfterVisible(".swatch-group.ordinal > div:nth-child(2)") - await processPause() - - await expectAttribute(".pie-slice._0 > path", "fill", "#ea5545") - await expectAttribute(".pie-slice._3 > path", "fill", "#87bc45") - await expectAttribute(".pie-slice._6 > path", "fill", "#ede15b") - await expectAttribute(".pie-slice._9 > path", "fill", "#ea5545") - - await clickAfterVisible(".color-swatch.selected") - await clickAfterVisible(".swatch-group.ordinal > div:nth-child(3)") - await processPause() - - await expectAttribute(".pie-slice._0 > path", "fill", "#22a7f0") - await expectAttribute(".pie-slice._3 > path", "fill", "#d4e666") - await expectAttribute(".pie-slice._6 > path", "fill", "#3ad6cd") - await expectAttribute(".pie-slice._9 > path", "fill", "#22a7f0") - - await clickAfterVisible(".color-swatch.selected") - await clickAfterVisible(".swatch-group.ordinal > div:nth-child(4)") - await processPause() - - await expectAttribute(".pie-slice._0 > path", "fill", "#ede15b") - await expectAttribute(".pie-slice._3 > path", "fill", "#27aeef") - await expectAttribute(".pie-slice._6 > path", "fill", "#ede15b") - await expectAttribute(".pie-slice._9 > path", "fill", "#ede15b") - - await clickAfterVisible(".color-swatch.selected") - await clickAfterVisible(".swatch-group.ordinal > div:nth-child(5)") - await processPause() - - await expectAttribute(".pie-slice._0 > path", "fill", "#27aeef") - await expectAttribute(".pie-slice._3 > path", "fill", "#ef9b20") - await expectAttribute(".pie-slice._6 > path", "fill", "#b33dc6") - await expectAttribute(".pie-slice._9 > path", "fill", "#bdcf32") + let previousColors = await getPieColors() + + for (let paletteIndex = 2; paletteIndex <= 5; paletteIndex += 1) { + await clickAfterVisible(".color-swatch.selected") + await clickAfterVisible( + `.swatch-group.ordinal > div:nth-child(${paletteIndex})` + ) + await processPause() + + const ordinalColors = await getPieColors() + expect(new Set(ordinalColors).size).toBeGreaterThan(1) + expect(ordinalColors).not.toEqual(previousColors) + previousColors = ordinalColors + } }) }) diff --git a/src/ui-tests/chart-creation/table-chart.ui.test.ts b/src/ui-tests/chart-creation/table-chart.ui.test.ts index 02a6974..06b8d22 100644 --- a/src/ui-tests/chart-creation/table-chart.ui.test.ts +++ b/src/ui-tests/chart-creation/table-chart.ui.test.ts @@ -8,7 +8,8 @@ import { saveChart, waitForVisible, setInputText, - clickAfterVisible + clickAfterVisible, + waitForCrossfilter } from "../utils" describe("Table chart", () => { @@ -90,9 +91,18 @@ describe("Table chart", () => { await saveChart() + const totalCount = await page.$eval(".count-selected", (element) => + Number(element.textContent.replace(/,/g, "")) + ) + await clickAfterVisible("table > tr:nth-child(2) td") + await waitForCrossfilter() - await expectContainsText(".count-selected", "67,280") + const selectedCount = await page.$eval(".count-selected", (element) => + Number(element.textContent.replace(/,/g, "")) + ) + expect(selectedCount).toBeGreaterThan(0) + expect(selectedCount).toBeLessThan(totalCount) await expect(page).toMatchElement('td[class="filtered cell-align-left"]') await expect(page).toMatchElement('[class="unfilter-btn"]') diff --git a/src/ui-tests/chart-creation/vega-combo/vega-combo-creation.ui.test.ts b/src/ui-tests/chart-creation/vega-combo/vega-combo-creation.ui.test.ts index 363b2ec..1cf4f4f 100644 --- a/src/ui-tests/chart-creation/vega-combo/vega-combo-creation.ui.test.ts +++ b/src/ui-tests/chart-creation/vega-combo/vega-combo-creation.ui.test.ts @@ -188,6 +188,11 @@ describe("Vega combo chart creation", () => { measures: ["airtime"] }) + await processPause(1) + const initialBarMarks = await page.$$( + ".vega-container .mark-rect.role-mark.bar > path" + ) + await expect(page).toClick("[data-testid='add-data-layer-button']") await processPause(1) @@ -203,9 +208,8 @@ describe("Vega combo chart creation", () => { ".vega-container .mark-rect.role-mark.bar > path" ) - // This is the number of base paths expected to appear for these two layers, - // 52 rows from flights and 51 rows from tweets - expect(numBarMarks.length).toEqual(103) + expect(initialBarMarks.length).toBeGreaterThan(0) + expect(numBarMarks.length).toBeGreaterThan(initialBarMarks.length) }) it("Creates a Vega bar chart with a binned date dimension", async () => { diff --git a/src/ui-tests/fixtures/heavydb/generate-fixtures.js b/src/ui-tests/fixtures/heavydb/generate-fixtures.js new file mode 100755 index 0000000..a7d19c4 --- /dev/null +++ b/src/ui-tests/fixtures/heavydb/generate-fixtures.js @@ -0,0 +1,303 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: Apache-2.0 + +const fs = require("fs") +const path = require("path") + +const outputDirectory = path.resolve( + process.argv[2] || path.join(__dirname, "generated") +) +// Fixture generation is a short-lived, sequential CI operation. +// eslint-disable-next-line no-sync +fs.mkdirSync(outputDirectory, { recursive: true }) + +const countries = [ + "US", + "BR", + "ID", + "AR", + "TR", + "GB", + "JP", + "MY", + "ES", + "PH", + "FR", + "SA", + "TH", + "RU", + "MX", + "CO", + "IT", + "PT", + "CA", + "UY", + "CL", + "IN", + "NL", + "ZA", + "VE", + "EG", + "AU", + "EC", + "SG", + "DE", + "IE", + "UA", + "PY", + "SE", + "AE", + "KW", + "NG", + "DO", + "PL", + "PR", + "PE", + "BY", + "BE", + "GT", + "KR", + "PK", + "CR", + "PA", + "LV", + "JO", + "IL" +] + +const destinations = [ + ["ATL", "Atlanta", "GA", -84.4277, 33.6407], + ["LAX", "Los Angeles", "CA", -118.4085, 33.9416], + ["ORD", "Chicago", "IL", -87.9073, 41.9742], + ["DFW", "Dallas", "TX", -97.0403, 32.8998], + ["DEN", "Denver", "CO", -104.6737, 39.8561], + ["JFK", "New York", "NY", -73.7781, 40.6413], + ["SFO", "San Francisco", "CA", -122.379, 37.6213], + ["SEA", "Seattle", "WA", -122.3088, 47.4502], + ["MIA", "Miami", "FL", -80.287, 25.7959], + ["BOS", "Boston", "MA", -71.0096, 42.3656], + ["PHX", "Phoenix", "AZ", -112.0116, 33.4342], + ["LAS", "Las Vegas", "NV", -115.1523, 36.08] +] + +const carriers = [ + ["American Airlines", "AA"], + ["Delta Air Lines", "DL"], + ["Southwest Airlines", "WN"], + ["United Airlines", "UA"] +] + +const csvCell = (value) => { + const text = String(value) + return /[",\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text +} + +const writeCsv = (name, columns, rows) => { + const body = [ + columns.join(","), + ...rows.map((row) => + columns.map((column) => csvCell(row[column])).join(",") + ) + ].join("\n") + // eslint-disable-next-line no-sync + fs.writeFileSync(path.join(outputDirectory, name), `${body}\n`) +} + +const flightColumns = [ + "carrier_name", + "dest", + "dest_city", + "dest_state", + "dest_country", + "dest_name", + "dest_lon", + "dest_lat", + "dest_merc_x", + "dest_merc_y", + "origin", + "origin_city", + "origin_state", + "origin_country", + "origin_name", + "origin_lon", + "origin_lat", + "origin_merc_x", + "origin_merc_y", + "airtime", + "taxiin", + "taxiout", + "carrierdelay", + "securitydelay", + "lateaircraftdelay", + "nasdelay", + "weatherdelay", + "arrtime", + "deptime", + "crsarrtime", + "crsdeptime", + "arrdelay", + "depdelay", + "crselapsedtime", + "actualelapsedtime", + "distance", + "flight_month", + "flight_dayofmonth", + "flight_dayofweek", + "flight_year", + "flightnum", + "uniquecarrier", + "tailnum", + "cancelled", + "cancellationcode", + "diverted", + "arr_timestamp", + "dep_timestamp", + "plane_aircraft_type", + "plane_engine_type", + "plane_issue_date", + "plane_manufacturer", + "plane_model", + "plane_status", + "plane_type", + "plane_year" +] + +const flightRows = Array.from({ length: 1200 }, (_, index) => { + const destination = destinations[index % destinations.length] + const origin = destinations[(index + 5) % destinations.length] + const carrier = carriers[index % carriers.length] + const month = (index % 12) + 1 + const day = (index % 28) + 1 + const airtime = 40 + (index % 181) + const departureHour = 5 + (index % 17) + const departureMinute = index % 60 + const departure = `${departureHour + .toString() + .padStart(2, "0")}:${departureMinute.toString().padStart(2, "0")}:00` + + return { + carrier_name: carrier[0], + dest: destination[0], + dest_city: destination[1], + dest_state: destination[2], + dest_country: "US", + dest_name: `${destination[1]} International`, + dest_lon: destination[3], + dest_lat: destination[4], + dest_merc_x: destination[3] * 100000, + dest_merc_y: destination[4] * 100000, + origin: origin[0], + origin_city: origin[1], + origin_state: origin[2], + origin_country: "US", + origin_name: `${origin[1]} International`, + origin_lon: origin[3], + origin_lat: origin[4], + origin_merc_x: origin[3] * 100000, + origin_merc_y: origin[4] * 100000, + airtime, + taxiin: 4 + (index % 17), + taxiout: 8 + (index % 23), + carrierdelay: index % 41, + securitydelay: index % 7, + lateaircraftdelay: index % 37, + nasdelay: index % 29, + weatherdelay: index % 13, + arrtime: ((departureHour + 2) % 24) * 100 + departureMinute, + deptime: departureHour * 100 + departureMinute, + crsarrtime: ((departureHour + 2) % 24) * 100, + crsdeptime: departureHour * 100, + arrdelay: (index % 61) - 20, + depdelay: (index % 47) - 15, + crselapsedtime: airtime + 35, + actualelapsedtime: airtime + 30 + (index % 12), + distance: 250 + (index % 2750), + flight_month: month, + flight_dayofmonth: day, + flight_dayofweek: (index % 7) + 1, + flight_year: 2025, + flightnum: 100 + index, + uniquecarrier: carrier[1], + tailnum: `N${(10000 + index).toString()}`, + cancelled: index % 113 === 0 ? "true" : "false", + cancellationcode: index % 113 === 0 ? "A" : "", + diverted: index % 197 === 0 ? "true" : "false", + arr_timestamp: `2025-${month + .toString() + .padStart(2, "0")}-${day.toString().padStart(2, "0")} ${( + (departureHour + 2) % + 24 + ) + .toString() + .padStart(2, "0")}:${departureMinute.toString().padStart(2, "0")}:00`, + dep_timestamp: `2025-${month + .toString() + .padStart(2, "0")}-${day.toString().padStart(2, "0")} ${departure}`, + plane_aircraft_type: "Fixed wing multi engine", + plane_engine_type: "Turbo-fan", + plane_issue_date: "2018-01-01", + plane_manufacturer: index % 2 === 0 ? "Boeing" : "Airbus", + plane_model: index % 2 === 0 ? "737" : "A320", + plane_status: "Valid", + plane_type: "Corporation", + plane_year: 2018 + } +}) + +const tweetRows = [] +countries.forEach((country, countryIndex) => { + const rowCount = 100 - countryIndex + for (let index = 0; index < rowCount; index += 1) { + const isCentralCluster = index % 4 === 0 + tweetRows.push({ + lon: isCentralCluster + ? -25 + (index % 10) * 1.5 + : -165 + ((countryIndex * 37 + index * 11) % 330), + lat: isCentralCluster + ? 20 + (index % 8) * 2 + : -65 + ((countryIndex * 17 + index * 7) % 130), + followees: 10 + ((countryIndex * 31 + index * 13) % 1000), + followers: 5 + ((countryIndex * 19 + index * 7) % 5000), + country, + state_abbr: destinations[(countryIndex + index) % destinations.length][2], + admin1: `Region ${(countryIndex % 10) + 1}`, + join_time: `2025-${((countryIndex % 12) + 1) + .toString() + .padStart(2, "0")}-${((index % 28) + 1) + .toString() + .padStart(2, "0")} 12:00:00` + }) + } +}) + +writeCsv("flights_donotmodify.csv", flightColumns, flightRows) +writeCsv( + "tweets_nov_feb.csv", + [ + "lon", + "lat", + "followees", + "followers", + "country", + "state_abbr", + "admin1", + "join_time" + ], + tweetRows +) +writeCsv( + "us_states_geo.csv", + ["NAME", "ALAND", "AWATER"], + [ + { NAME: "California", ALAND: 403466232000, AWATER: 20253000000 }, + { NAME: "Texas", ALAND: 676587800000, AWATER: 19092000000 }, + { NAME: "Florida", ALAND: 138887400000, AWATER: 31424000000 }, + { NAME: "New York", ALAND: 122057000000, AWATER: 19240000000 }, + { NAME: "Washington", ALAND: 172119000000, AWATER: 12542000000 } + ] +) + +// eslint-disable-next-line no-console +console.log( + `Generated ${flightRows.length} flights, ${tweetRows.length} tweets, and 5 states in ${outputDirectory}` +) diff --git a/src/ui-tests/fixtures/heavydb/schema.sql b/src/ui-tests/fixtures/heavydb/schema.sql new file mode 100644 index 0000000..36bf35d --- /dev/null +++ b/src/ui-tests/fixtures/heavydb/schema.sql @@ -0,0 +1,87 @@ +DROP TABLE IF EXISTS flights_donotmodify; +CREATE TABLE flights_donotmodify ( + carrier_name TEXT ENCODING DICT(32), + dest TEXT ENCODING DICT(32), + dest_city TEXT ENCODING DICT(32), + dest_state TEXT ENCODING DICT(32), + dest_country TEXT ENCODING DICT(32), + dest_name TEXT ENCODING DICT(32), + dest_lon DOUBLE, + dest_lat DOUBLE, + dest_merc_x DOUBLE, + dest_merc_y DOUBLE, + origin TEXT ENCODING DICT(32), + origin_city TEXT ENCODING DICT(32), + origin_state TEXT ENCODING DICT(32), + origin_country TEXT ENCODING DICT(32), + origin_name TEXT ENCODING DICT(32), + origin_lon DOUBLE, + origin_lat DOUBLE, + origin_merc_x DOUBLE, + origin_merc_y DOUBLE, + airtime SMALLINT, + taxiin SMALLINT, + taxiout SMALLINT, + carrierdelay SMALLINT, + securitydelay SMALLINT, + lateaircraftdelay SMALLINT, + nasdelay SMALLINT, + weatherdelay SMALLINT, + arrtime SMALLINT, + deptime SMALLINT, + crsarrtime SMALLINT, + crsdeptime SMALLINT, + arrdelay SMALLINT, + depdelay SMALLINT, + crselapsedtime SMALLINT, + actualelapsedtime SMALLINT, + distance INT, + flight_month SMALLINT, + flight_dayofmonth SMALLINT, + flight_dayofweek SMALLINT, + flight_year SMALLINT, + flightnum INT, + uniquecarrier TEXT ENCODING DICT(32), + tailnum TEXT ENCODING DICT(32), + cancelled BOOLEAN, + cancellationcode TEXT ENCODING DICT(32), + diverted BOOLEAN, + arr_timestamp TIMESTAMP(0), + dep_timestamp TIMESTAMP(0), + plane_aircraft_type TEXT ENCODING DICT(32), + plane_engine_type TEXT ENCODING DICT(32), + plane_issue_date DATE, + plane_manufacturer TEXT ENCODING DICT(32), + plane_model TEXT ENCODING DICT(32), + plane_status TEXT ENCODING DICT(32), + plane_type TEXT ENCODING DICT(32), + plane_year SMALLINT +); +COPY flights_donotmodify +FROM '/var/lib/heavyai/storage/import/immerse-ci/flights_donotmodify.csv' +WITH (header = 'true'); + +DROP TABLE IF EXISTS tweets_nov_feb; +CREATE TABLE tweets_nov_feb ( + lon DOUBLE, + lat DOUBLE, + followees INT, + followers INT, + country TEXT ENCODING DICT(32), + state_abbr TEXT ENCODING DICT(32), + admin1 TEXT ENCODING DICT(32), + join_time TIMESTAMP(0) +); +COPY tweets_nov_feb +FROM '/var/lib/heavyai/storage/import/immerse-ci/tweets_nov_feb.csv' +WITH (header = 'true'); + +DROP TABLE IF EXISTS us_states_geo; +CREATE TABLE us_states_geo ( + NAME TEXT ENCODING DICT(32), + ALAND BIGINT, + AWATER BIGINT +); +COPY us_states_geo +FROM '/var/lib/heavyai/storage/import/immerse-ci/us_states_geo.csv' +WITH (header = 'true'); diff --git a/src/ui-tests/utils/dashboards.ts b/src/ui-tests/utils/dashboards.ts index c103ff7..ecf67ee 100644 --- a/src/ui-tests/utils/dashboards.ts +++ b/src/ui-tests/utils/dashboards.ts @@ -9,6 +9,7 @@ import { } from "./common" import { DASHBOARD_TITLE_PLACEHOLDER_ID } from "../../constants/dashboards" import { Page } from "@playwright/test" +import * as path from "path" export async function clearDashboards(dashboardSubstring: string) { const firstDashboardInList = "#dashboards-list .row-item:first-of-type" @@ -131,6 +132,26 @@ export async function goToDashboardList() { return await page.goto(BASE_TEST_URL, { waitUntil: "networkidle0" }) } +export async function importDashboard(dashboardPath: string) { + const searchInput = '[data-testid="dashboard-search-bar-field"]' + const importButton = '[data-testid="import-dashboard-button"]' + const fileInput = '.import-dialog input[type="file"]' + + await page.click(searchInput, { clickCount: 3 }) + await page.keyboard.press("Backspace") + await clickAfterVisible(importButton) + await page.waitForSelector(fileInput) + + const uploadInput = await page.$(fileInput) + await uploadInput.uploadFile(path.resolve(dashboardPath)) + await page.waitForSelector(".import-message-success") + await clickAfterVisible( + ".import-dialog .mdc-dialog__actions .import-dialog-action-button:last-child" + ) + await page.waitForSelector(".import-dialog", { hidden: true }) + await processPause() +} + export async function loadDashboard(dashboardName: string) { await goToDashboardList() await searchDashboardList(dashboardName) From a3bf76b69562c11109432cc97cede3518c47a17a Mon Sep 17 00:00:00 2001 From: Joe O'Hallaron Date: Mon, 28 Sep 2026 22:15:34 -0600 Subject: [PATCH 07/11] fix: scope runner temp paths to workflow steps --- .github/workflows/immerse-puppeteer-automation.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/immerse-puppeteer-automation.yml b/.github/workflows/immerse-puppeteer-automation.yml index 274759a..70dfebb 100644 --- a/.github/workflows/immerse-puppeteer-automation.yml +++ b/.github/workflows/immerse-puppeteer-automation.yml @@ -29,7 +29,6 @@ jobs: timeout-minutes: 120 env: HEAVYDB_RESOURCE_SUFFIX: ${{ github.run_id }}-${{ github.run_attempt }} - HEAVYDB_PRODUCT_DIR: ${{ runner.temp }}/heavydb-product-${{ github.run_id }}-${{ github.run_attempt }} HEAVYDB_DIAGNOSTICS_DIR: ${{ github.workspace }}/heavydb-diagnostics steps: - name: Checkout @@ -73,6 +72,7 @@ jobs: id: heavydb-product env: HEAVYAI_BINARY_SCAN_TOKEN: ${{ secrets.HEAVYAI_BINARY_SCAN_TOKEN }} + HEAVYDB_PRODUCT_DIR: ${{ runner.temp }}/heavydb-product-${{ github.run_id }}-${{ github.run_attempt }} HEAVYDB_RUN_ID: ${{ inputs.heavydb_run_id }} run: | args=(--output-dir "$HEAVYDB_PRODUCT_DIR") @@ -159,6 +159,8 @@ jobs: - name: Remove disposable HeavyDB resources if: always() + env: + HEAVYDB_PRODUCT_DIR: ${{ runner.temp }}/heavydb-product-${{ github.run_id }}-${{ github.run_attempt }} run: | if [ -x scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh ]; then scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh From a5a9ca11801d309a82fde0106c770395a17c34d0 Mon Sep 17 00:00:00 2001 From: Joe O'Hallaron Date: Mon, 28 Sep 2026 22:20:22 -0600 Subject: [PATCH 08/11] fix: fall back to existing artifact credential --- .github/workflows/immerse-puppeteer-automation.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/immerse-puppeteer-automation.yml b/.github/workflows/immerse-puppeteer-automation.yml index 70dfebb..41a5dbe 100644 --- a/.github/workflows/immerse-puppeteer-automation.yml +++ b/.github/workflows/immerse-puppeteer-automation.yml @@ -71,7 +71,9 @@ jobs: - name: Download and verify HeavyDB product id: heavydb-product env: - HEAVYAI_BINARY_SCAN_TOKEN: ${{ secrets.HEAVYAI_BINARY_SCAN_TOKEN }} + # PRIVATE_PACKAGES_TOKEN is a migration fallback for the private + # binary-scan repository only; npm installation uses no token. + HEAVYAI_BINARY_SCAN_TOKEN: ${{ secrets.HEAVYAI_BINARY_SCAN_TOKEN || secrets.PRIVATE_PACKAGES_TOKEN }} HEAVYDB_PRODUCT_DIR: ${{ runner.temp }}/heavydb-product-${{ github.run_id }}-${{ github.run_attempt }} HEAVYDB_RUN_ID: ${{ inputs.heavydb_run_id }} run: | From 28148479491bf3a723c417f17b26bb6893cacf6f Mon Sep 17 00:00:00 2001 From: Joe O'Hallaron Date: Mon, 28 Sep 2026 22:26:48 -0600 Subject: [PATCH 09/11] fix: report inaccessible product bundles --- .github/workflows/immerse-puppeteer-automation.yml | 4 +--- .../ci/puppeteer-heavydb/download-heavydb-artifact.sh | 10 ++++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/immerse-puppeteer-automation.yml b/.github/workflows/immerse-puppeteer-automation.yml index 41a5dbe..70dfebb 100644 --- a/.github/workflows/immerse-puppeteer-automation.yml +++ b/.github/workflows/immerse-puppeteer-automation.yml @@ -71,9 +71,7 @@ jobs: - name: Download and verify HeavyDB product id: heavydb-product env: - # PRIVATE_PACKAGES_TOKEN is a migration fallback for the private - # binary-scan repository only; npm installation uses no token. - HEAVYAI_BINARY_SCAN_TOKEN: ${{ secrets.HEAVYAI_BINARY_SCAN_TOKEN || secrets.PRIVATE_PACKAGES_TOKEN }} + HEAVYAI_BINARY_SCAN_TOKEN: ${{ secrets.HEAVYAI_BINARY_SCAN_TOKEN }} HEAVYDB_PRODUCT_DIR: ${{ runner.temp }}/heavydb-product-${{ github.run_id }}-${{ github.run_attempt }} HEAVYDB_RUN_ID: ${{ inputs.heavydb_run_id }} run: | diff --git a/scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh b/scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh index 46de6ce..bdca5b5 100755 --- a/scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh +++ b/scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh @@ -73,7 +73,10 @@ resolve_artifact() { local candidate if [[ -n "${requested_run_id}" ]]; then - find_artifact_for_run "${requested_run_id}" + if ! find_artifact_for_run "${requested_run_id}"; then + echo "Run ${requested_run_id} has no accessible published bundle; verify HEAVYAI_BINARY_SCAN_TOKEN can read ${SCAN_REPOSITORY}" >&2 + return 1 + fi return fi @@ -91,7 +94,10 @@ resolve_artifact() { return 1 } -IFS=$'\t' read -r run_id run_date artifact_name < <(resolve_artifact) +if ! resolved_artifact="$(resolve_artifact)"; then + exit 1 +fi +IFS=$'\t' read -r run_id run_date artifact_name <<<"${resolved_artifact}" artifact_relative_path="runs/${run_date}/${run_id}/${artifact_name}" mkdir -p "${output_dir}" From 3bad2bac7c97a06245e0223089219e4d5738229b Mon Sep 17 00:00:00 2001 From: Joe O'Hallaron Date: Tue, 29 Sep 2026 10:52:38 -0600 Subject: [PATCH 10/11] fix: remove puppeteer actions stuff for now, will add in follow up --- .../immerse-puppeteer-automation.yml | 167 ------------------ scripts/ci/puppeteer-heavydb/Dockerfile | 37 ---- .../ci/puppeteer-heavydb/cleanup-heavydb.sh | 21 --- .../puppeteer-heavydb/collect-diagnostics.sh | 24 --- .../download-heavydb-artifact.sh | 162 ----------------- scripts/ci/puppeteer-heavydb/entrypoint.sh | 52 ------ scripts/ci/puppeteer-heavydb/load-fixtures.sh | 43 ----- scripts/ci/puppeteer-heavydb/start-heavydb.sh | 64 ------- src/ui-tests/README-Puppeteer-UI-tests.md | 27 +-- 9 files changed, 9 insertions(+), 588 deletions(-) delete mode 100644 .github/workflows/immerse-puppeteer-automation.yml delete mode 100644 scripts/ci/puppeteer-heavydb/Dockerfile delete mode 100755 scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh delete mode 100755 scripts/ci/puppeteer-heavydb/collect-diagnostics.sh delete mode 100755 scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh delete mode 100755 scripts/ci/puppeteer-heavydb/entrypoint.sh delete mode 100755 scripts/ci/puppeteer-heavydb/load-fixtures.sh delete mode 100755 scripts/ci/puppeteer-heavydb/start-heavydb.sh diff --git a/.github/workflows/immerse-puppeteer-automation.yml b/.github/workflows/immerse-puppeteer-automation.yml deleted file mode 100644 index 70dfebb..0000000 --- a/.github/workflows/immerse-puppeteer-automation.yml +++ /dev/null @@ -1,167 +0,0 @@ -name: immerse-puppeteer-automation - -# Temporarily manual-only until the disposable HeavyDB path has a successful run. -# Replaces the legacy Jenkins immerse-puppeteer-automation job. - -on: - workflow_dispatch: - inputs: - ref: - description: "Branch or SHA to test" - type: string - default: "" - heavydb_run_id: - description: "Product Builder run ID (leave empty for latest published master build)" - type: string - default: "" - -permissions: - contents: read - -concurrency: - group: immerse-puppeteer-automation-${{ github.ref }} - cancel-in-progress: true - -jobs: - immerse-puppeteer-automation: - name: immerse-puppeteer-automation - runs-on: linux-amd64-cpu32 - timeout-minutes: 120 - env: - HEAVYDB_RESOURCE_SUFFIX: ${{ github.run_id }}-${{ github.run_attempt }} - HEAVYDB_DIAGNOSTICS_DIR: ${{ github.workspace }}/heavydb-diagnostics - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - persist-credentials: false - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - - name: Upgrade npm - run: npm install --global npm@^11.6.1 - - - name: Use HTTPS for GitHub dependencies - run: | - git config --global --add \ - url."https://github.com/".insteadOf \ - "git@github.com:" - git config --global --add \ - url."https://github.com/".insteadOf \ - "ssh://git@github.com/" - - - name: Install dependencies - run: npm ci - - - name: Check artifact download prerequisites - run: | - git lfs version - jq --version - unzip -v - - - name: Install cosign - uses: sigstore/cosign-installer@v3 - - - name: Download and verify HeavyDB product - id: heavydb-product - env: - HEAVYAI_BINARY_SCAN_TOKEN: ${{ secrets.HEAVYAI_BINARY_SCAN_TOKEN }} - HEAVYDB_PRODUCT_DIR: ${{ runner.temp }}/heavydb-product-${{ github.run_id }}-${{ github.run_attempt }} - HEAVYDB_RUN_ID: ${{ inputs.heavydb_run_id }} - run: | - args=(--output-dir "$HEAVYDB_PRODUCT_DIR") - if [ -n "$HEAVYDB_RUN_ID" ]; then - args+=(--run-id "$HEAVYDB_RUN_ID") - fi - scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh "${args[@]}" - - - name: Start disposable HeavyDB - env: - HEAVYDB_TARBALL: ${{ steps.heavydb-product.outputs.tarball }} - run: scripts/ci/puppeteer-heavydb/start-heavydb.sh - - - name: Load deterministic test fixtures - run: scripts/ci/puppeteer-heavydb/load-fixtures.sh - - - name: Configure test server - run: | - node <<'NODE' - const fs = require("fs") - const server = { - database: "heavyai", - master: true, - username: "admin", - password: "HyperInteractive", - url: "http://127.0.0.1:6273/", - GTM: "GTM-MDD888", - theme: "dark" - } - - fs.writeFileSync( - "src/servers.local.json", - `${JSON.stringify([server], null, 2)}\n` - ) - NODE - - - name: Start Immerse - run: | - npm run start -- --no-open > webpack.log 2>&1 & - echo "WEBPACK_SERVER_PID=$!" >> "$GITHUB_ENV" - - - name: Wait for Immerse - run: | - for attempt in {1..120}; do - if curl --fail --silent --show-error http://localhost:8002/ > /dev/null; then - exit 0 - fi - sleep 5 - done - - cat webpack.log - echo "Immerse did not become ready within 10 minutes" >&2 - exit 1 - - - name: Run Puppeteer integration tests - run: npm run test:ui - - - name: Stop Immerse - if: always() - run: | - if [ -n "${WEBPACK_SERVER_PID:-}" ]; then - kill "$WEBPACK_SERVER_PID" 2>/dev/null || true - fi - - - name: Collect HeavyDB diagnostics - if: always() - run: | - if [ -x scripts/ci/puppeteer-heavydb/collect-diagnostics.sh ]; then - scripts/ci/puppeteer-heavydb/collect-diagnostics.sh - fi - - - name: Upload test diagnostics - if: always() - uses: actions/upload-artifact@v4 - with: - name: puppeteer-test-diagnostics-${{ github.run_attempt }} - path: | - heavydb-diagnostics - jest-results.xml - webpack.log - src/ui-tests/_screenshot-*.png - if-no-files-found: ignore - retention-days: 14 - - - name: Remove disposable HeavyDB resources - if: always() - env: - HEAVYDB_PRODUCT_DIR: ${{ runner.temp }}/heavydb-product-${{ github.run_id }}-${{ github.run_attempt }} - run: | - if [ -x scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh ]; then - scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh - fi diff --git a/scripts/ci/puppeteer-heavydb/Dockerfile b/scripts/ci/puppeteer-heavydb/Dockerfile deleted file mode 100644 index b68ae53..0000000 --- a/scripts/ci/puppeteer-heavydb/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM ubuntu:22.04 AS extract - -WORKDIR /opt/heavyai -COPY product.tar.gz /tmp/product.tar.gz -RUN tar xzf /tmp/product.tar.gz --strip-components=1 -C /opt/heavyai - -FROM nvcr.io/nvidia/cuda:12.9.1-runtime-ubuntu22.04 - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - bsdmainutils \ - ca-certificates \ - clang-14 \ - curl \ - default-jre-headless \ - libegl1 \ - libglx0 \ - liblz4-tool \ - libopengl0 \ - libpython3.10 \ - procps \ - python3.10 \ - python3.10-venv \ - zip \ - && rm -rf /var/lib/apt/lists/* - -COPY --from=extract /opt/heavyai /opt/heavyai -COPY entrypoint.sh /opt/heavyai/ci-entrypoint.sh -RUN chmod +x /opt/heavyai/ci-entrypoint.sh - -WORKDIR /opt/heavyai -EXPOSE 6273 6274 6278 6279 - -HEALTHCHECK --interval=5s --timeout=5s --start-period=20s --retries=24 \ - CMD printf 'SELECT 1;\n' | /opt/heavyai/bin/heavysql heavyai -u admin -p HyperInteractive >/dev/null - -ENTRYPOINT ["/opt/heavyai/ci-entrypoint.sh"] diff --git a/scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh b/scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh deleted file mode 100755 index 28da87c..0000000 --- a/scripts/ci/puppeteer-heavydb/cleanup-heavydb.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -: "${HEAVYDB_RESOURCE_SUFFIX:?HEAVYDB_RESOURCE_SUFFIX is required}" - -resource_suffix="$(tr -cd '[:alnum:]_-' <<<"${HEAVYDB_RESOURCE_SUFFIX}")" -readonly container_name="immerse-puppeteer-heavydb-${resource_suffix}" -readonly image_name="immerse-puppeteer-heavydb:${resource_suffix}" -readonly volume_name="immerse-puppeteer-heavydb-${resource_suffix}" -readonly build_context="${RUNNER_TEMP:-/tmp}/immerse-heavydb-build-${resource_suffix}" - -docker rm --force "${container_name}" 2>/dev/null || true -docker volume rm --force "${volume_name}" 2>/dev/null || true -docker image rm --force "${image_name}" 2>/dev/null || true -rm -rf "${build_context}" - -if [[ -n "${HEAVYDB_PRODUCT_DIR:-}" ]]; then - rm -rf "${HEAVYDB_PRODUCT_DIR}" -fi diff --git a/scripts/ci/puppeteer-heavydb/collect-diagnostics.sh b/scripts/ci/puppeteer-heavydb/collect-diagnostics.sh deleted file mode 100755 index 97b5c16..0000000 --- a/scripts/ci/puppeteer-heavydb/collect-diagnostics.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -: "${HEAVYDB_RESOURCE_SUFFIX:?HEAVYDB_RESOURCE_SUFFIX is required}" -: "${HEAVYDB_DIAGNOSTICS_DIR:?HEAVYDB_DIAGNOSTICS_DIR is required}" - -resource_suffix="$(tr -cd '[:alnum:]_-' <<<"${HEAVYDB_RESOURCE_SUFFIX}")" -readonly container_name="immerse-puppeteer-heavydb-${resource_suffix}" - -mkdir -p "${HEAVYDB_DIAGNOSTICS_DIR}" -if ! docker inspect "${container_name}" >"${HEAVYDB_DIAGNOSTICS_DIR}/container-inspect.json" 2>&1; then - echo "Container ${container_name} does not exist; no HeavyDB diagnostics to collect" - exit 0 -fi - -docker logs "${container_name}" >"${HEAVYDB_DIAGNOSTICS_DIR}/container.log" 2>&1 || true -docker cp \ - "${container_name}:/var/lib/heavyai/ci-logs" \ - "${HEAVYDB_DIAGNOSTICS_DIR}/ci-logs" 2>/dev/null || true -docker cp \ - "${container_name}:/var/lib/heavyai/storage/log" \ - "${HEAVYDB_DIAGNOSTICS_DIR}/heavydb-logs" 2>/dev/null || true diff --git a/scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh b/scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh deleted file mode 100755 index bdca5b5..0000000 --- a/scripts/ci/puppeteer-heavydb/download-heavydb-artifact.sh +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -readonly HEAVYDB_REPOSITORY="heavyai/heavydb" -readonly PRODUCT_WORKFLOW="rc-builder.yml" -readonly SCAN_REPOSITORY="heavyai/heavyai-binary-scan-testing" -readonly ARTIFACT_PATTERN='^heavyai-.*-ubuntu22\.04-x86_64-render-signed\.zip$' -readonly MINIMUM_ARTIFACT_BYTES=100000000 - -output_dir="" -requested_run_id="" - -while (($#)); do - case "$1" in - --output-dir) - output_dir="$2" - shift 2 - ;; - --run-id) - requested_run_id="$2" - shift 2 - ;; - *) - echo "Unknown argument: $1" >&2 - exit 2 - ;; - esac -done - -: "${output_dir:?--output-dir is required}" -: "${HEAVYAI_BINARY_SCAN_TOKEN:?HEAVYAI_BINARY_SCAN_TOKEN is required}" - -api() { - curl --fail --silent --show-error \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${HEAVYAI_BINARY_SCAN_TOKEN}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/$1" -} - -find_artifact_for_run() { - local run_id="$1" - local run_json run_date directory_json artifact_name - - run_json="$(api "repos/${HEAVYDB_REPOSITORY}/actions/runs/${run_id}")" - if [[ "$(jq -r '.path' <<<"${run_json}")" != ".github/workflows/${PRODUCT_WORKFLOW}" ]]; then - echo "Run ${run_id} is not a Product Builder run" >&2 - return 1 - fi - if [[ "$(jq -r '.head_branch' <<<"${run_json}")" != "master" ]] || - [[ "$(jq -r '.conclusion' <<<"${run_json}")" != "success" ]]; then - echo "Run ${run_id} is not a successful master build" >&2 - return 1 - fi - - run_date="$(jq -r '.created_at[0:10]' <<<"${run_json}")" - if ! directory_json="$(api "repos/${SCAN_REPOSITORY}/contents/runs/${run_date}/${run_id}" 2>/dev/null)"; then - return 1 - fi - - artifact_name="$( - jq -r '.[].name' <<<"${directory_json}" | - awk -v pattern="${ARTIFACT_PATTERN}" '$0 ~ pattern { print; exit }' - )" - [[ -n "${artifact_name}" ]] || return 1 - - printf '%s\t%s\t%s\n' "${run_id}" "${run_date}" "${artifact_name}" -} - -resolve_artifact() { - local candidate - - if [[ -n "${requested_run_id}" ]]; then - if ! find_artifact_for_run "${requested_run_id}"; then - echo "Run ${requested_run_id} has no accessible published bundle; verify HEAVYAI_BINARY_SCAN_TOKEN can read ${SCAN_REPOSITORY}" >&2 - return 1 - fi - return - fi - - while IFS= read -r candidate; do - if find_artifact_for_run "${candidate}"; then - return - fi - echo "Product Builder run ${candidate} has no published Ubuntu x86_64 bundle; trying the previous run" >&2 - done < <( - api "repos/${HEAVYDB_REPOSITORY}/actions/workflows/${PRODUCT_WORKFLOW}/runs?branch=master&status=success&per_page=100" | - jq -r '.workflow_runs[].id' - ) - - echo "No successful Product Builder run with a published Ubuntu x86_64 bundle was found" >&2 - return 1 -} - -if ! resolved_artifact="$(resolve_artifact)"; then - exit 1 -fi -IFS=$'\t' read -r run_id run_date artifact_name <<<"${resolved_artifact}" -artifact_relative_path="runs/${run_date}/${run_id}/${artifact_name}" - -mkdir -p "${output_dir}" -output_dir="$(cd "${output_dir}" && pwd)" -checkout_dir="${output_dir}/scan-repository" -bundle_dir="${output_dir}/bundle" - -cleanup_credentials() { - if [[ -d "${checkout_dir}/.git" ]]; then - git -C "${checkout_dir}" remote set-url origin "https://github.com/${SCAN_REPOSITORY}.git" || true - fi -} -trap cleanup_credentials EXIT - -rm -rf "${checkout_dir}" "${bundle_dir}" -GIT_LFS_SKIP_SMUDGE=1 git clone --filter=blob:none --no-checkout \ - "https://x-access-token:${HEAVYAI_BINARY_SCAN_TOKEN}@github.com/${SCAN_REPOSITORY}.git" \ - "${checkout_dir}" -git -C "${checkout_dir}" sparse-checkout init --cone -git -C "${checkout_dir}" sparse-checkout set "runs/${run_date}/${run_id}" -git -C "${checkout_dir}" checkout -git -C "${checkout_dir}" lfs pull --include="${artifact_relative_path}" --exclude="" -cleanup_credentials - -artifact_path="${checkout_dir}/${artifact_relative_path}" -artifact_size="$(wc -c <"${artifact_path}" | tr -d ' ')" -if ((artifact_size < MINIMUM_ARTIFACT_BYTES)); then - echo "Downloaded artifact is only ${artifact_size} bytes; Git LFS content was not retrieved" >&2 - exit 1 -fi - -mkdir -p "${bundle_dir}" -unzip -q "${artifact_path}" -d "${bundle_dir}" -tarball_path="$( - find "${bundle_dir}" -maxdepth 1 -type f -name '*.tar.gz' -print -quit -)" -if [[ -z "${tarball_path}" ]]; then - echo "Signed bundle does not contain a product tarball" >&2 - exit 1 -fi -bundle_path="${tarball_path}.bundle" -if [[ ! -f "${bundle_path}" ]]; then - echo "Signed bundle does not contain $(basename "${bundle_path}")" >&2 - exit 1 -fi - -cosign verify-blob \ - --bundle="${bundle_path}" \ - --certificate-identity-regexp='https://github.com/heavyai/heavydb/.github/workflows/rc-builder.yml@refs/heads/.*' \ - --certificate-oidc-issuer='https://token.actions.githubusercontent.com' \ - "${tarball_path}" - -rm -rf "${checkout_dir}" - -echo "Verified HeavyDB product from run ${run_id}: $(basename "${tarball_path}")" -if [[ -n "${GITHUB_OUTPUT:-}" ]]; then - { - echo "run_id=${run_id}" - echo "artifact_name=${artifact_name}" - echo "tarball=${tarball_path}" - } >>"${GITHUB_OUTPUT}" -fi diff --git a/scripts/ci/puppeteer-heavydb/entrypoint.sh b/scripts/ci/puppeteer-heavydb/entrypoint.sh deleted file mode 100755 index b6a23b9..0000000 --- a/scripts/ci/puppeteer-heavydb/entrypoint.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -readonly data_dir="${HEAVYDB_DATA_DIR:-/var/lib/heavyai/storage}" -readonly console_log_dir="/var/lib/heavyai/ci-logs" - -mkdir -p "${console_log_dir}" -if [[ ! -d "${data_dir}/mapd_catalogs" ]]; then - mkdir -p "${data_dir}" - /opt/heavyai/bin/initheavy -f --data "${data_dir}" -fi - -children=() -stop_services() { - trap - SIGINT SIGTERM EXIT - if ((${#children[@]})); then - kill "${children[@]}" 2>/dev/null || true - wait "${children[@]}" 2>/dev/null || true - fi -} -trap stop_services SIGINT SIGTERM EXIT - -/opt/heavyai/bin/heavydb "${data_dir}" \ - --cpu-only \ - --port 6274 \ - --http-port 6278 \ - --calcite-port 6279 \ - >"${console_log_dir}/heavydb-console.log" 2>&1 & -children+=("$!") -echo "HeavyDB started as PID ${children[-1]}" - -/opt/heavyai/bin/heavy_web_server \ - --port 6273 \ - --backend-url http://127.0.0.1:6278 \ - --data "${data_dir}" \ - >"${console_log_dir}/heavy-web-server-console.log" 2>&1 & -children+=("$!") -echo "Heavy web server started as PID ${children[-1]}" - -set +e -wait -n "${children[@]}" -status=$? -set -e - -echo "A HeavyDB service exited with status ${status}" >&2 -for log_file in "${console_log_dir}"/*.log; do - echo "===== ${log_file} =====" >&2 - cat "${log_file}" >&2 -done -exit "${status}" diff --git a/scripts/ci/puppeteer-heavydb/load-fixtures.sh b/scripts/ci/puppeteer-heavydb/load-fixtures.sh deleted file mode 100755 index 53f10c8..0000000 --- a/scripts/ci/puppeteer-heavydb/load-fixtures.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -: "${HEAVYDB_RESOURCE_SUFFIX:?HEAVYDB_RESOURCE_SUFFIX is required}" - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -repository_root="$(cd "${script_dir}/../../.." && pwd)" -resource_suffix="$(tr -cd '[:alnum:]_-' <<<"${HEAVYDB_RESOURCE_SUFFIX}")" -readonly container_name="immerse-puppeteer-heavydb-${resource_suffix}" -readonly generated_dir="${RUNNER_TEMP:-/tmp}/immerse-heavydb-fixtures-${resource_suffix}" -readonly container_fixture_dir="/var/lib/heavyai/storage/import/immerse-ci" -readonly heavysql="/opt/heavyai/bin/heavysql heavyai -u admin -p HyperInteractive" - -rm -rf "${generated_dir}" -node \ - "${repository_root}/src/ui-tests/fixtures/heavydb/generate-fixtures.js" \ - "${generated_dir}" - -docker exec "${container_name}" mkdir -p "${container_fixture_dir}" -docker cp "${generated_dir}/." "${container_name}:${container_fixture_dir}" -docker exec -i "${container_name}" bash -c "${heavysql}" < \ - "${repository_root}/src/ui-tests/fixtures/heavydb/schema.sql" - -validation="$( - docker exec "${container_name}" bash -c "${heavysql}" <<'SQL' -SELECT CASE WHEN - (SELECT COUNT(*) FROM flights_donotmodify) = 1200 - AND (SELECT COUNT(*) FROM flights_donotmodify WHERE flight_month = 100) = 0 - AND (SELECT COUNT(*) FROM flights_donotmodify WHERE flight_month IN (5, 12)) > 0 - AND (SELECT COUNT(DISTINCT country) FROM tweets_nov_feb) = 51 - AND (SELECT COUNT(*) FROM us_states_geo) = 5 -THEN 'PASS' ELSE 'FAIL' END AS fixture_status; -SQL -)" -echo "${validation}" -if [[ "${validation}" != *PASS* ]]; then - echo "HeavyDB fixture validation failed" >&2 - exit 1 -fi - -rm -rf "${generated_dir}" diff --git a/scripts/ci/puppeteer-heavydb/start-heavydb.sh b/scripts/ci/puppeteer-heavydb/start-heavydb.sh deleted file mode 100755 index 4894aa8..0000000 --- a/scripts/ci/puppeteer-heavydb/start-heavydb.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -: "${HEAVYDB_TARBALL:?HEAVYDB_TARBALL is required}" -: "${HEAVYDB_RESOURCE_SUFFIX:?HEAVYDB_RESOURCE_SUFFIX is required}" - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -resource_suffix="$(tr -cd '[:alnum:]_-' <<<"${HEAVYDB_RESOURCE_SUFFIX}")" -readonly container_name="immerse-puppeteer-heavydb-${resource_suffix}" -readonly image_name="immerse-puppeteer-heavydb:${resource_suffix}" -readonly volume_name="immerse-puppeteer-heavydb-${resource_suffix}" -readonly build_context="${RUNNER_TEMP:-/tmp}/immerse-heavydb-build-${resource_suffix}" - -rm -rf "${build_context}" -mkdir -p "${build_context}" -cp "${HEAVYDB_TARBALL}" "${build_context}/product.tar.gz" -cp "${script_dir}/Dockerfile" "${script_dir}/entrypoint.sh" "${build_context}/" - -docker build --tag "${image_name}" "${build_context}" -docker volume create "${volume_name}" >/dev/null -docker run --detach \ - --name "${container_name}" \ - --volume "${volume_name}:/var/lib/heavyai" \ - --publish 127.0.0.1:6273:6273 \ - --publish 127.0.0.1:6274:6274 \ - --publish 127.0.0.1:6278:6278 \ - --publish 127.0.0.1:6279:6279 \ - "${image_name}" >/dev/null - -ready=false -for _ in $(seq 1 90); do - if ! docker inspect "${container_name}" --format '{{.State.Running}}' | grep -q true; then - echo "HeavyDB container exited before becoming ready" >&2 - docker logs "${container_name}" >&2 || true - exit 1 - fi - - if docker exec "${container_name}" bash -c \ - "printf 'SELECT 1;\\n' | /opt/heavyai/bin/heavysql heavyai -u admin -p HyperInteractive >/dev/null" && - curl --fail --silent --show-error http://127.0.0.1:6273/ >/dev/null; then - ready=true - break - fi - sleep 2 -done - -if [[ "${ready}" != "true" ]]; then - echo "HeavyDB did not become ready within 180 seconds" >&2 - docker logs "${container_name}" >&2 || true - exit 1 -fi - -rm -rf "${build_context}" -echo "Disposable HeavyDB is ready at http://127.0.0.1:6273" - -if [[ -n "${GITHUB_OUTPUT:-}" ]]; then - { - echo "container=${container_name}" - echo "image=${image_name}" - echo "volume=${volume_name}" - } >>"${GITHUB_OUTPUT}" -fi diff --git a/src/ui-tests/README-Puppeteer-UI-tests.md b/src/ui-tests/README-Puppeteer-UI-tests.md index fa74098..7b4909e 100644 --- a/src/ui-tests/README-Puppeteer-UI-tests.md +++ b/src/ui-tests/README-Puppeteer-UI-tests.md @@ -2,24 +2,15 @@ ## Configuration requirements -CI starts a disposable HeavyDB and loads deterministic CSV fixtures generated -by `fixtures/heavydb/generate-fixtures.js`. Local runs may use the same loader -or a server containing the following contract: - -- `flights_donotmodify` has 1,200 synthetic flight rows. It contains the - standard flight columns used throughout the tests, including carrier, - destination, origin, delay, date, location, and plane metadata. Months 5 and - 12 are present, month 100 is absent, and carrier values include American - Airlines, Delta Air Lines, Southwest Airlines, and United Airlines. -- `tweets_nov_feb` has deterministic coordinate clusters and all 51 country - codes asserted by the scatter and point-map tests. It also includes - `followees`, `followers`, `state_abbr`, `admin1`, and `join_time`. -- `us_states_geo` has five state rows with `NAME`, `ALAND`, and `AWATER` - columns. The active tests use it for SQL view creation and deletion. - -The schema is defined in `fixtures/heavydb/schema.sql`. Update the generator, -schema, validation in `scripts/ci/puppeteer-heavydb/load-fixtures.sh`, and this -contract together when a UI test requires another table or column. +The UI tests require connecting to a server running a HeavyDB instance with +specific data sources: + +- flights_donotmodify +- us_states_geo +- tweets_nov_feb + +This list of required data sources must be updated if new data sources are +incorporated into UI tests. ## How to run in the background From 765090b0149c15b7b5ba29e254eeeb6f7adecea5 Mon Sep 17 00:00:00 2001 From: Joe O'Hallaron Date: Tue, 29 Sep 2026 10:55:11 -0600 Subject: [PATCH 11/11] fix: remove more puppeteer implementation stuff --- .../fixtures/heavydb/generate-fixtures.js | 303 ------------------ src/ui-tests/fixtures/heavydb/schema.sql | 87 ----- 2 files changed, 390 deletions(-) delete mode 100755 src/ui-tests/fixtures/heavydb/generate-fixtures.js delete mode 100644 src/ui-tests/fixtures/heavydb/schema.sql diff --git a/src/ui-tests/fixtures/heavydb/generate-fixtures.js b/src/ui-tests/fixtures/heavydb/generate-fixtures.js deleted file mode 100755 index a7d19c4..0000000 --- a/src/ui-tests/fixtures/heavydb/generate-fixtures.js +++ /dev/null @@ -1,303 +0,0 @@ -#!/usr/bin/env node -// SPDX-License-Identifier: Apache-2.0 - -const fs = require("fs") -const path = require("path") - -const outputDirectory = path.resolve( - process.argv[2] || path.join(__dirname, "generated") -) -// Fixture generation is a short-lived, sequential CI operation. -// eslint-disable-next-line no-sync -fs.mkdirSync(outputDirectory, { recursive: true }) - -const countries = [ - "US", - "BR", - "ID", - "AR", - "TR", - "GB", - "JP", - "MY", - "ES", - "PH", - "FR", - "SA", - "TH", - "RU", - "MX", - "CO", - "IT", - "PT", - "CA", - "UY", - "CL", - "IN", - "NL", - "ZA", - "VE", - "EG", - "AU", - "EC", - "SG", - "DE", - "IE", - "UA", - "PY", - "SE", - "AE", - "KW", - "NG", - "DO", - "PL", - "PR", - "PE", - "BY", - "BE", - "GT", - "KR", - "PK", - "CR", - "PA", - "LV", - "JO", - "IL" -] - -const destinations = [ - ["ATL", "Atlanta", "GA", -84.4277, 33.6407], - ["LAX", "Los Angeles", "CA", -118.4085, 33.9416], - ["ORD", "Chicago", "IL", -87.9073, 41.9742], - ["DFW", "Dallas", "TX", -97.0403, 32.8998], - ["DEN", "Denver", "CO", -104.6737, 39.8561], - ["JFK", "New York", "NY", -73.7781, 40.6413], - ["SFO", "San Francisco", "CA", -122.379, 37.6213], - ["SEA", "Seattle", "WA", -122.3088, 47.4502], - ["MIA", "Miami", "FL", -80.287, 25.7959], - ["BOS", "Boston", "MA", -71.0096, 42.3656], - ["PHX", "Phoenix", "AZ", -112.0116, 33.4342], - ["LAS", "Las Vegas", "NV", -115.1523, 36.08] -] - -const carriers = [ - ["American Airlines", "AA"], - ["Delta Air Lines", "DL"], - ["Southwest Airlines", "WN"], - ["United Airlines", "UA"] -] - -const csvCell = (value) => { - const text = String(value) - return /[",\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text -} - -const writeCsv = (name, columns, rows) => { - const body = [ - columns.join(","), - ...rows.map((row) => - columns.map((column) => csvCell(row[column])).join(",") - ) - ].join("\n") - // eslint-disable-next-line no-sync - fs.writeFileSync(path.join(outputDirectory, name), `${body}\n`) -} - -const flightColumns = [ - "carrier_name", - "dest", - "dest_city", - "dest_state", - "dest_country", - "dest_name", - "dest_lon", - "dest_lat", - "dest_merc_x", - "dest_merc_y", - "origin", - "origin_city", - "origin_state", - "origin_country", - "origin_name", - "origin_lon", - "origin_lat", - "origin_merc_x", - "origin_merc_y", - "airtime", - "taxiin", - "taxiout", - "carrierdelay", - "securitydelay", - "lateaircraftdelay", - "nasdelay", - "weatherdelay", - "arrtime", - "deptime", - "crsarrtime", - "crsdeptime", - "arrdelay", - "depdelay", - "crselapsedtime", - "actualelapsedtime", - "distance", - "flight_month", - "flight_dayofmonth", - "flight_dayofweek", - "flight_year", - "flightnum", - "uniquecarrier", - "tailnum", - "cancelled", - "cancellationcode", - "diverted", - "arr_timestamp", - "dep_timestamp", - "plane_aircraft_type", - "plane_engine_type", - "plane_issue_date", - "plane_manufacturer", - "plane_model", - "plane_status", - "plane_type", - "plane_year" -] - -const flightRows = Array.from({ length: 1200 }, (_, index) => { - const destination = destinations[index % destinations.length] - const origin = destinations[(index + 5) % destinations.length] - const carrier = carriers[index % carriers.length] - const month = (index % 12) + 1 - const day = (index % 28) + 1 - const airtime = 40 + (index % 181) - const departureHour = 5 + (index % 17) - const departureMinute = index % 60 - const departure = `${departureHour - .toString() - .padStart(2, "0")}:${departureMinute.toString().padStart(2, "0")}:00` - - return { - carrier_name: carrier[0], - dest: destination[0], - dest_city: destination[1], - dest_state: destination[2], - dest_country: "US", - dest_name: `${destination[1]} International`, - dest_lon: destination[3], - dest_lat: destination[4], - dest_merc_x: destination[3] * 100000, - dest_merc_y: destination[4] * 100000, - origin: origin[0], - origin_city: origin[1], - origin_state: origin[2], - origin_country: "US", - origin_name: `${origin[1]} International`, - origin_lon: origin[3], - origin_lat: origin[4], - origin_merc_x: origin[3] * 100000, - origin_merc_y: origin[4] * 100000, - airtime, - taxiin: 4 + (index % 17), - taxiout: 8 + (index % 23), - carrierdelay: index % 41, - securitydelay: index % 7, - lateaircraftdelay: index % 37, - nasdelay: index % 29, - weatherdelay: index % 13, - arrtime: ((departureHour + 2) % 24) * 100 + departureMinute, - deptime: departureHour * 100 + departureMinute, - crsarrtime: ((departureHour + 2) % 24) * 100, - crsdeptime: departureHour * 100, - arrdelay: (index % 61) - 20, - depdelay: (index % 47) - 15, - crselapsedtime: airtime + 35, - actualelapsedtime: airtime + 30 + (index % 12), - distance: 250 + (index % 2750), - flight_month: month, - flight_dayofmonth: day, - flight_dayofweek: (index % 7) + 1, - flight_year: 2025, - flightnum: 100 + index, - uniquecarrier: carrier[1], - tailnum: `N${(10000 + index).toString()}`, - cancelled: index % 113 === 0 ? "true" : "false", - cancellationcode: index % 113 === 0 ? "A" : "", - diverted: index % 197 === 0 ? "true" : "false", - arr_timestamp: `2025-${month - .toString() - .padStart(2, "0")}-${day.toString().padStart(2, "0")} ${( - (departureHour + 2) % - 24 - ) - .toString() - .padStart(2, "0")}:${departureMinute.toString().padStart(2, "0")}:00`, - dep_timestamp: `2025-${month - .toString() - .padStart(2, "0")}-${day.toString().padStart(2, "0")} ${departure}`, - plane_aircraft_type: "Fixed wing multi engine", - plane_engine_type: "Turbo-fan", - plane_issue_date: "2018-01-01", - plane_manufacturer: index % 2 === 0 ? "Boeing" : "Airbus", - plane_model: index % 2 === 0 ? "737" : "A320", - plane_status: "Valid", - plane_type: "Corporation", - plane_year: 2018 - } -}) - -const tweetRows = [] -countries.forEach((country, countryIndex) => { - const rowCount = 100 - countryIndex - for (let index = 0; index < rowCount; index += 1) { - const isCentralCluster = index % 4 === 0 - tweetRows.push({ - lon: isCentralCluster - ? -25 + (index % 10) * 1.5 - : -165 + ((countryIndex * 37 + index * 11) % 330), - lat: isCentralCluster - ? 20 + (index % 8) * 2 - : -65 + ((countryIndex * 17 + index * 7) % 130), - followees: 10 + ((countryIndex * 31 + index * 13) % 1000), - followers: 5 + ((countryIndex * 19 + index * 7) % 5000), - country, - state_abbr: destinations[(countryIndex + index) % destinations.length][2], - admin1: `Region ${(countryIndex % 10) + 1}`, - join_time: `2025-${((countryIndex % 12) + 1) - .toString() - .padStart(2, "0")}-${((index % 28) + 1) - .toString() - .padStart(2, "0")} 12:00:00` - }) - } -}) - -writeCsv("flights_donotmodify.csv", flightColumns, flightRows) -writeCsv( - "tweets_nov_feb.csv", - [ - "lon", - "lat", - "followees", - "followers", - "country", - "state_abbr", - "admin1", - "join_time" - ], - tweetRows -) -writeCsv( - "us_states_geo.csv", - ["NAME", "ALAND", "AWATER"], - [ - { NAME: "California", ALAND: 403466232000, AWATER: 20253000000 }, - { NAME: "Texas", ALAND: 676587800000, AWATER: 19092000000 }, - { NAME: "Florida", ALAND: 138887400000, AWATER: 31424000000 }, - { NAME: "New York", ALAND: 122057000000, AWATER: 19240000000 }, - { NAME: "Washington", ALAND: 172119000000, AWATER: 12542000000 } - ] -) - -// eslint-disable-next-line no-console -console.log( - `Generated ${flightRows.length} flights, ${tweetRows.length} tweets, and 5 states in ${outputDirectory}` -) diff --git a/src/ui-tests/fixtures/heavydb/schema.sql b/src/ui-tests/fixtures/heavydb/schema.sql deleted file mode 100644 index 36bf35d..0000000 --- a/src/ui-tests/fixtures/heavydb/schema.sql +++ /dev/null @@ -1,87 +0,0 @@ -DROP TABLE IF EXISTS flights_donotmodify; -CREATE TABLE flights_donotmodify ( - carrier_name TEXT ENCODING DICT(32), - dest TEXT ENCODING DICT(32), - dest_city TEXT ENCODING DICT(32), - dest_state TEXT ENCODING DICT(32), - dest_country TEXT ENCODING DICT(32), - dest_name TEXT ENCODING DICT(32), - dest_lon DOUBLE, - dest_lat DOUBLE, - dest_merc_x DOUBLE, - dest_merc_y DOUBLE, - origin TEXT ENCODING DICT(32), - origin_city TEXT ENCODING DICT(32), - origin_state TEXT ENCODING DICT(32), - origin_country TEXT ENCODING DICT(32), - origin_name TEXT ENCODING DICT(32), - origin_lon DOUBLE, - origin_lat DOUBLE, - origin_merc_x DOUBLE, - origin_merc_y DOUBLE, - airtime SMALLINT, - taxiin SMALLINT, - taxiout SMALLINT, - carrierdelay SMALLINT, - securitydelay SMALLINT, - lateaircraftdelay SMALLINT, - nasdelay SMALLINT, - weatherdelay SMALLINT, - arrtime SMALLINT, - deptime SMALLINT, - crsarrtime SMALLINT, - crsdeptime SMALLINT, - arrdelay SMALLINT, - depdelay SMALLINT, - crselapsedtime SMALLINT, - actualelapsedtime SMALLINT, - distance INT, - flight_month SMALLINT, - flight_dayofmonth SMALLINT, - flight_dayofweek SMALLINT, - flight_year SMALLINT, - flightnum INT, - uniquecarrier TEXT ENCODING DICT(32), - tailnum TEXT ENCODING DICT(32), - cancelled BOOLEAN, - cancellationcode TEXT ENCODING DICT(32), - diverted BOOLEAN, - arr_timestamp TIMESTAMP(0), - dep_timestamp TIMESTAMP(0), - plane_aircraft_type TEXT ENCODING DICT(32), - plane_engine_type TEXT ENCODING DICT(32), - plane_issue_date DATE, - plane_manufacturer TEXT ENCODING DICT(32), - plane_model TEXT ENCODING DICT(32), - plane_status TEXT ENCODING DICT(32), - plane_type TEXT ENCODING DICT(32), - plane_year SMALLINT -); -COPY flights_donotmodify -FROM '/var/lib/heavyai/storage/import/immerse-ci/flights_donotmodify.csv' -WITH (header = 'true'); - -DROP TABLE IF EXISTS tweets_nov_feb; -CREATE TABLE tweets_nov_feb ( - lon DOUBLE, - lat DOUBLE, - followees INT, - followers INT, - country TEXT ENCODING DICT(32), - state_abbr TEXT ENCODING DICT(32), - admin1 TEXT ENCODING DICT(32), - join_time TIMESTAMP(0) -); -COPY tweets_nov_feb -FROM '/var/lib/heavyai/storage/import/immerse-ci/tweets_nov_feb.csv' -WITH (header = 'true'); - -DROP TABLE IF EXISTS us_states_geo; -CREATE TABLE us_states_geo ( - NAME TEXT ENCODING DICT(32), - ALAND BIGINT, - AWATER BIGINT -); -COPY us_states_geo -FROM '/var/lib/heavyai/storage/import/immerse-ci/us_states_geo.csv' -WITH (header = 'true');