From 458530c902898a892dbe4dec4ba1e9d3fb091cfc Mon Sep 17 00:00:00 2001 From: woksin Date: Sun, 25 Oct 2026 05:49:00 +0100 Subject: [PATCH 1/4] Publish specification reports and summaries from CI Vitest writes JUnit reports only when SPEC_REPORTS_DIR is set, so local runs are unchanged. .NET specification results are now uploaded on every run with a 3-day retention and summarized in the job summary. --- .github/scripts/summarize-spec-reports.mjs | 80 ++++++++++++++++ .../scripts/summarize-spec-reports.test.mjs | 91 +++++++++++++++++++ .github/workflows/dotnet-build.yml | 11 ++- .gitignore | 1 + vite.base.ts | 7 ++ 5 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/summarize-spec-reports.mjs create mode 100644 .github/scripts/summarize-spec-reports.test.mjs diff --git a/.github/scripts/summarize-spec-reports.mjs b/.github/scripts/summarize-spec-reports.mjs new file mode 100644 index 00000000..5542fc2f --- /dev/null +++ b/.github/scripts/summarize-spec-reports.mjs @@ -0,0 +1,80 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import { spawnSync } from 'node:child_process'; +import { existsSync, readdirSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +function readReport(path) { + // Python's XML parser is also required by the existing specification runner. + const parser = `import json, sys, xml.etree.ElementTree as ET +root = ET.parse(sys.argv[1]).getroot() +names = ('testsuite', 'Counters', 'UnitTestResult', 'Times') +print(json.dumps({name: [e.attrib for e in root.iter() if e.tag.split('}')[-1] == name] for name in names}))`; + const result = spawnSync('python3', ['-c', parser, path], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }); + if (result.error || result.status !== 0) throw new Error(`Cannot parse ${path}: ${result.error?.message ?? result.stderr.trim()}`); + return JSON.parse(result.stdout); +} + +function seconds(duration = '0') { + return duration.split(':').reduce((total, part) => total * 60 + Number(part), 0); +} + +function cell(value) { + return String(value).replace(/\|/g, '\\|').replace(/[\r\n]/g, ' '); +} + +function junit(document, name) { + const suites = document.testsuite; + if (!suites.length) throw new Error(`No test suites found in ${name}`); + const totals = suites.reduce((sum, suite) => ({ + tests: sum.tests + Number(suite.tests ?? 0), + failed: sum.failed + Number(suite.failures ?? 0) + Number(suite.errors ?? 0), + skipped: sum.skipped + Number(suite.skipped ?? 0), + duration: sum.duration + seconds(suite.time) + }), { tests: 0, failed: 0, skipped: 0, duration: 0 }); + return { name, ...totals, slow: suites.map(suite => ({ name: suite.name ?? name, duration: seconds(suite.time) })) }; +} + +function trx(document, name) { + if (document.Counters.length !== 1) throw new Error(`Expected one TRX counter in ${name}`); + const counters = document.Counters[0]; + const results = document.UnitTestResult; + const times = document.Times[0] ?? {}; + const slow = results.map(result => ({ name: result.testName ?? name, duration: seconds(result.duration) })); + return { + name, + tests: Number(counters.total ?? results.length), + failed: Number(counters.failed ?? 0) + Number(counters.error ?? 0) + Number(counters.timeout ?? 0) + Number(counters.aborted ?? 0), + skipped: Number(counters.notExecuted ?? 0), + duration: times.start && times.finish ? (Date.parse(times.finish) - Date.parse(times.start)) / 1000 : slow.reduce((sum, test) => sum + test.duration, 0), + slow + }; +} + +const directory = process.argv[2]; +const files = directory && existsSync(directory) + ? readdirSync(directory, { recursive: true }).filter(file => /(?:\.junit\.xml|\.trx)$/.test(file)).sort() + : []; + +if (!files.length) { + console.error(`No specification reports found in ${directory ?? ''}`); + process.exitCode = 2; +} else try { + const reports = files.map(file => { + const path = join(directory, file); + const document = readReport(path); + return file.endsWith('.trx') ? trx(document, relative(directory, path)) : junit(document, basename(file, '.junit.xml')); + }); + console.log('## Specification results\n\n| Workspace / project | Tests | Failed | Skipped | Seconds |\n| --- | ---: | ---: | ---: | ---: |'); + for (const report of reports) { + console.log(`| ${cell(report.name)} | ${report.tests} | ${report.failed} | ${report.skipped} | ${report.duration.toFixed(3)} |`); + } + console.log('\n### Slowest 10 files / tests\n\n| Report | File / test | Seconds |\n| --- | --- | ---: |'); + const slowest = reports.flatMap(report => report.slow.map(item => ({ ...item, report: report.name }))) + .sort((left, right) => right.duration - left.duration).slice(0, 10); + for (const item of slowest) console.log(`| ${cell(item.report)} | ${cell(item.name)} | ${item.duration.toFixed(3)} |`); +} catch (error) { + console.error(error.message); + process.exitCode = 2; +} diff --git a/.github/scripts/summarize-spec-reports.test.mjs b/.github/scripts/summarize-spec-reports.test.mjs new file mode 100644 index 00000000..2659080a --- /dev/null +++ b/.github/scripts/summarize-spec-reports.test.mjs @@ -0,0 +1,91 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { test } from 'node:test'; + +const scripts = dirname(fileURLToPath(import.meta.url)); +const root = resolve(scripts, '../..'); + +function fixture(t) { + mkdirSync(join(root, '.ai-work/262'), { recursive: true }); + const directory = mkdtempSync(join(root, '.ai-work/262/summary-')); + t.after(() => rmSync(directory, { recursive: true })); + return directory; +} + +function summarize(directory) { + const result = spawnSync(process.execPath, [join(scripts, 'summarize-spec-reports.mjs'), directory], { encoding: 'utf8' }); + assert.ifError(result.error); + return result; +} + +test('summarizes nested JUnit reports without counting the root twice, even on test failure', t => { + const directory = fixture(t); + mkdirSync(join(directory, 'nested')); + writeFileSync(join(directory, 'nested/model.junit.xml'), ''); + const result = summarize(directory); + assert.equal(result.status, 0); + assert.match(result.stdout, /\| model \| 3 \| 1 \| 1 \| 3\.000 \|/); + assert.match(result.stdout, /slow&file\\\|name \| 2\.500/); + assert.ok(result.stdout.indexOf('slow&file') < result.stdout.indexOf('| fast |')); +}); + +test('summarizes TRX counters, elapsed time and per-test duration', t => { + const directory = fixture(t); + writeFileSync(join(directory, 'Engine.Specs.trx'), ''); + const result = summarize(directory); + assert.equal(result.status, 0); + assert.match(result.stdout, /Engine.Specs.trx \| 3 \| 1 \| 1 \| 63\.000/); + assert.match(result.stdout, /slow \| 62\.500/); +}); + +test('keeps the slowest ten entries across reports', t => { + const directory = fixture(t); + const suites = Array.from({ length: 12 }, (_, index) => ``).join(''); + writeFileSync(join(directory, 'many.junit.xml'), `${suites}`); + const result = summarize(directory); + assert.equal(result.status, 0); + assert.match(result.stdout, /file-11 \| 11\.000/); + assert.doesNotMatch(result.stdout, /\| file-[01] \|/); +}); + +test('parses XML quoting and namespace prefixes rather than extracting attributes by regex', t => { + const directory = fixture(t); + writeFileSync(join(directory, 'quoted.junit.xml'), ""); + const result = summarize(directory); + assert.equal(result.status, 0); + assert.match(result.stdout, /quoted \| 1 \| 0 \| 0 \| 0\.500/); + assert.match(result.stdout, /one > zero/); +}); + +test('refuses incomplete or malformed XML instead of reporting partial results', t => { + const directory = fixture(t); + for (const xml of ['', '', '']) { + writeFileSync(join(directory, 'broken.junit.xml'), xml); + const result = summarize(directory); + assert.equal(result.status, 2); + assert.match(result.stderr, /Cannot parse/); + assert.equal(result.stdout, ''); + } +}); + +test('refuses XML without the expected report structure', t => { + const directory = fixture(t); + for (const file of ['empty.junit.xml', 'empty.trx']) { + const report = join(directory, file); + writeFileSync(report, ''); + assert.equal(summarize(directory).status, 2); + rmSync(report); + } +}); + +test('refuses an empty report directory', t => { + const result = summarize(fixture(t)); + assert.equal(result.status, 2); + assert.match(result.stderr, /No specification reports found/); +}); diff --git a/.github/workflows/dotnet-build.yml b/.github/workflows/dotnet-build.yml index 11e1d1ee..9579b486 100644 --- a/.github/workflows/dotnet-build.yml +++ b/.github/workflows/dotnet-build.yml @@ -28,6 +28,8 @@ on: - "Source/DotNET/**" - ".github/scripts/run-specifications.sh" - ".github/scripts/run-specifications.test.mjs" + - ".github/scripts/summarize-spec-reports.mjs" + - ".github/scripts/summarize-spec-reports.test.mjs" - "Directory.Build.props" - "Directory.Packages.props" - "global.json" @@ -134,10 +136,15 @@ jobs: - name: Run specification project (${{ matrix.project }}) run: bash .github/scripts/run-specifications.sh Release ${{ matrix.project }} - - name: Upload failed specification results - if: failure() + - name: Upload specification results + if: always() uses: actions/upload-artifact@v4 with: name: specification-results-${{ strategy.job-index }} path: TestResults/ + retention-days: 3 if-no-files-found: ignore + + - name: Summarize specification results + if: always() + run: node .github/scripts/summarize-spec-reports.mjs TestResults >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 5a7e48bb..2dec44b9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore .buildsettings +spec-reports/ # User-specific files *.suo diff --git a/vite.base.ts b/vite.base.ts index cae555a7..5617f770 100644 --- a/vite.base.ts +++ b/vite.base.ts @@ -1,6 +1,7 @@ // Copyright (c) Cratis. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +import { basename, join } from 'node:path'; import commonjs from 'vite-plugin-commonjs'; import tsconfigPaths from 'vite-tsconfig-paths'; @@ -15,6 +16,12 @@ export function createConfig() { }, }, test: { + ...(process.env.SPEC_REPORTS_DIR ? { + reporters: ['default', 'junit'], + outputFile: { + junit: join(process.env.SPEC_REPORTS_DIR, `${basename(process.cwd())}${process.env.SPEC_REPORT_SUFFIX ?? ''}.junit.xml`) + } + } : {}), globals: true, environment: 'node', sourcemap: false, From 684947906b3179bf962e9f36bf12c8f0bf7b419c Mon Sep 17 00:00:00 2001 From: woksin Date: Sun, 25 Oct 2026 05:49:00 +0100 Subject: [PATCH 2/4] Split browser specifications into three CI shards Shards are assigned from the discovered browser workspaces and run against the built artifacts, uploading their JUnit reports for the summary. --- .github/scripts/browser-spec-shards.mjs | 49 +++++++ .github/scripts/browser-spec-shards.test.mjs | 94 ++++++++++++++ .github/workflows/javascript-build.yml | 129 ++++++++++++++++--- 3 files changed, 253 insertions(+), 19 deletions(-) create mode 100644 .github/scripts/browser-spec-shards.mjs create mode 100644 .github/scripts/browser-spec-shards.test.mjs diff --git a/.github/scripts/browser-spec-shards.mjs b/.github/scripts/browser-spec-shards.mjs new file mode 100644 index 00000000..fc50b5b6 --- /dev/null +++ b/.github/scripts/browser-spec-shards.mjs @@ -0,0 +1,49 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import { readFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { globSync } from 'glob'; + +// Local baseline wall-time medians (seconds); replace with CI measurements when available. +const weights = { + '@cratis/scene.primereact': 31.37, + '@cratis/scene.blueprint.default': 43.24, + '@cratis/scene.blueprint.administration': 27.17, + '@cratis/scene.blueprint.crm': 94.65 +}; + +function browserWorkspaces(root) { + const { workspaces } = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')); + const ignore = ['**/node_modules/**', ...workspaces.filter(pattern => pattern.startsWith('!')).map(pattern => pattern.slice(1))]; + const patterns = workspaces.filter(pattern => !pattern.startsWith('!')).map(pattern => `${pattern}/package.json`); + return globSync(patterns, { cwd: root, ignore, nodir: true }) + .map(file => JSON.parse(readFileSync(join(root, file), 'utf8'))) + .filter(manifest => manifest.scripts?.['test:browser']) + .map(manifest => ({ name: manifest.name, weight: weights[manifest.name] ?? 10 })) + .sort((left, right) => right.weight - left.weight || left.name.localeCompare(right.name)); +} + +export function browserSpecShards(root, count = 3) { + if (!Number.isInteger(count) || count < 1) throw new Error('Shard count must be a positive integer'); + const workspaces = browserWorkspaces(root); + if (!workspaces.length) throw new Error('No browser specification workspaces found'); + const shards = Array.from({ length: Math.min(count, workspaces.length) }, (_, index) => ({ shard: index + 1, workspaces: [], weight: 0 })); + for (const workspace of workspaces) { + const lightest = shards.reduce((best, shard) => shard.weight < best.weight ? shard : best); + lightest.workspaces.push(workspace.name); + lightest.weight += workspace.weight; + } + return shards.map(({ shard, workspaces }) => ({ shard, workspaces: workspaces.join(' ') })); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); + console.log(JSON.stringify(browserSpecShards(root, Number(process.argv[2] ?? 3)))); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/.github/scripts/browser-spec-shards.test.mjs b/.github/scripts/browser-spec-shards.test.mjs new file mode 100644 index 00000000..d06e50fa --- /dev/null +++ b/.github/scripts/browser-spec-shards.test.mjs @@ -0,0 +1,94 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { test } from 'node:test'; +import { browserSpecShards } from './browser-spec-shards.mjs'; + +const scripts = dirname(fileURLToPath(import.meta.url)); +const root = resolve(scripts, '../..'); + +test('assigns every real browser workspace exactly once across at most three nonempty shards', () => { + const directory = join(root, 'Source/JavaScript'); + const expected = readdirSync(directory).map(name => JSON.parse(readFileSync(join(directory, name, 'package.json'), 'utf8'))) + .filter(manifest => manifest.scripts?.['test:browser']).map(manifest => manifest.name).sort(); + const shards = browserSpecShards(root); + const actual = shards.flatMap(shard => shard.workspaces.split(' ')); + assert.ok(expected.length > 0); + assert.ok(shards.length > 0 && shards.length <= 3); + assert.equal(new Set(actual).size, actual.length); + assert.deepEqual(actual.sort(), expected); + assert.deepEqual(shards.map(shard => shard.shard), [1, 2, 3]); +}); + +function fixture(t, count = 0) { + mkdirSync(join(root, '.ai-work/262'), { recursive: true }); + const directory = mkdtempSync(join(root, '.ai-work/262/shards-')); + t.after(() => rmSync(directory, { recursive: true })); + writeFileSync(join(directory, 'package.json'), JSON.stringify({ workspaces: ['Source/JavaScript/**/*', '!**/dist/**', '!**/obj/**', '!**/bin/**'] })); + mkdirSync(join(directory, 'Source/JavaScript'), { recursive: true }); + for (let index = 0; index < count; index++) { + const workspace = join(directory, `Source/JavaScript/workspace${index}`); + mkdirSync(workspace); + writeFileSync(join(workspace, 'package.json'), JSON.stringify({ name: `@test/workspace${index}`, scripts: { 'test:browser': 'vitest' } })); + } + return directory; +} + +test('uses deterministic greedy balancing and does not create empty shards', t => { + const directory = fixture(t, 2); + assert.deepEqual(browserSpecShards(directory), [ + { shard: 1, workspaces: '@test/workspace0' }, + { shard: 2, workspaces: '@test/workspace1' } + ]); + assert.deepEqual(browserSpecShards(directory, 1), [{ shard: 1, workspaces: '@test/workspace0 @test/workspace1' }]); + assert.throws(() => browserSpecShards(directory, 0), /positive integer/); +}); + +test('discovers nested declared workspaces but not build output or installed dependencies', t => { + const directory = fixture(t, 1); + for (const name of ['nested/browser', 'nested/dist/cjs', 'nested/obj/cache', 'nested/bin/Debug', 'node_modules/dependency']) { + const workspace = join(directory, 'Source/JavaScript', name); + mkdirSync(workspace, { recursive: true }); + writeFileSync(join(workspace, 'package.json'), JSON.stringify({ name, scripts: { 'test:browser': 'vitest' } })); + } + const actual = browserSpecShards(directory).flatMap(shard => shard.workspaces.split(' ')).sort(); + assert.deepEqual(actual, ['@test/workspace0', 'nested/browser']); +}); + +test('CLI prints compact JSON for the discovered workspaces', () => { + const result = spawnSync(process.execPath, [join(scripts, 'browser-spec-shards.mjs'), '3'], { encoding: 'utf8' }); + assert.equal(result.status, 0); + assert.equal(result.stdout.trim(), JSON.stringify(browserSpecShards(root))); +}); + +test('the workflow runs every workspace in a shard and preserves a failure', t => { + const directory = fixture(t); + const yarn = join(directory, 'yarn'); + writeFileSync(yarn, '#!/bin/bash\nprintf "%s\\n" "$2" >> "$CALLS"\n[[ "$2" != "$FAILED_WORKSPACE" ]]\n', { mode: 0o755 }); + const workflow = readFileSync(join(root, '.github/workflows/javascript-build.yml'), 'utf8'); + const loop = workflow.match(/ run: \|\n( failed=\(\)[\s\S]*?)\n\n/)[1].replace(/^ /gm, ''); + for (const failed of ['', '@test/first']) { + const calls = join(directory, `calls-${failed ? 'failed' : 'passed'}`); + const result = spawnSync('bash', ['-e', '-c', loop], { + encoding: 'utf8', + env: { ...process.env, PATH: `${directory}:${process.env.PATH}`, CALLS: calls, FAILED_WORKSPACE: failed, WORKSPACES: '@test/first @test/second' } + }); + assert.equal(result.status, failed ? 1 : 0, result.stderr); + assert.equal(readFileSync(calls, 'utf8'), '@test/first\n@test/second\n'); + } +}); + +test('CLI fails rather than silently accepting an empty discovery', t => { + const directory = fixture(t); + mkdirSync(join(directory, '.github/scripts'), { recursive: true }); + const script = join(directory, '.github/scripts/browser-spec-shards.mjs'); + copyFileSync(join(scripts, 'browser-spec-shards.mjs'), script); + const result = spawnSync(process.execPath, [script], { encoding: 'utf8' }); + assert.equal(result.status, 1); + assert.match(result.stderr, /No browser specification workspaces found/); +}); diff --git a/.github/workflows/javascript-build.yml b/.github/workflows/javascript-build.yml index 12457e6d..edf1d862 100644 --- a/.github/workflows/javascript-build.yml +++ b/.github/workflows/javascript-build.yml @@ -21,11 +21,18 @@ on: - "**" paths: - "Source/JavaScript/**" + - ".github/workflows/javascript-build.yml" + - ".github/scripts/summarize-spec-reports.*" + - ".github/scripts/browser-spec-shards.*" + - "vite.base.ts" + - "vitest.browser.global-setup.ts" jobs: build: runs-on: ubuntu-latest timeout-minutes: 30 + outputs: + browser-shards: ${{ steps.browser-shards.outputs.matrix }} steps: - name: Checkout code @@ -50,10 +57,21 @@ jobs: - name: Yarn install run: yarn + - name: Verify specification reporting and sharding + run: node --test .github/scripts/summarize-spec-reports.test.mjs .github/scripts/browser-spec-shards.test.mjs + + - name: Discover browser specification shards + id: browser-shards + run: | + matrix=$(node .github/scripts/browser-spec-shards.mjs 3) + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + - name: Build JS/TS run: | export NODE_OPTIONS="--max-old-space-size=4096" yarn ci + env: + SPEC_REPORTS_DIR: ${{ github.workspace }}/spec-reports - name: Verify packed packages hold no specifications run: yarn verify-packages @@ -65,6 +83,8 @@ jobs: # The generator all domain blueprints start from must keep producing a package that is done, without edits. - name: Scaffold a blueprint and verify it + env: + SPEC_REPORTS_DIR: ${{ github.workspace }}/spec-reports run: | # The cache holds every node_modules folder, including one a previous run left in the scaffold's folder. rm -rf Source/JavaScript/blueprint.ci-scaffold @@ -82,27 +102,98 @@ jobs: - name: Build Storybook run: yarn build-storybook - # The foundation components are audited in a real Chromium at desktop, tablet and phone widths in both themes, - # with colour contrast, which jsdom cannot judge (spec Part B 6). A regression fails the build. + - name: Archive JavaScript build output + run: tar -czf js-dist.tar.gz Source/JavaScript/*/dist + + - name: Upload JavaScript build output + uses: actions/upload-artifact@v4 + with: + name: js-dist + path: js-dist.tar.gz + retention-days: 1 + if-no-files-found: error + + - name: Upload specification reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: spec-reports-unit + path: spec-reports/ + retention-days: 3 + if-no-files-found: ignore + + - name: Summarize specification reports + if: always() + run: node .github/scripts/summarize-spec-reports.mjs spec-reports >> "$GITHUB_STEP_SUMMARY" + + browser-specs: + needs: [build] + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.build.outputs.browser-shards) }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: 23.x + registry-url: "https://registry.npmjs.org" + + - uses: actions/cache@v4 + id: yarn-cache + with: + path: | + .yarn/cache + **/node_modules + **/.eslintcache + **/yarn.lock + key: ${{ runner.os }}-yarn-${{ hashFiles('**/package.json') }} + + - name: Yarn install + run: yarn + + - name: Download JavaScript build output + uses: actions/download-artifact@v4 + with: + name: js-dist + + - name: Extract JavaScript build output + run: tar -xzf js-dist.tar.gz + + # Audit every blueprint in real Chromium, including contrast, which jsdom cannot judge. - name: Install Chromium run: npx playwright-core install --with-deps chromium - name: Browser specifications + env: + WORKSPACES: ${{ matrix.workspaces }} + SPEC_REPORTS_DIR: ${{ github.workspace }}/spec-reports + SPEC_REPORT_SUFFIX: .browser run: | - yarn workspace @cratis/scene.primereact test:browser - yarn workspace @cratis/scene.blueprint.default test:browser - yarn workspace @cratis/scene.blueprint.crm test:browser - yarn workspace @cratis/scene.blueprint.ticketing test:browser - yarn workspace @cratis/scene.blueprint.approvals test:browser - yarn workspace @cratis/scene.blueprint.hr test:browser - yarn workspace @cratis/scene.blueprint.inventory test:browser - yarn workspace @cratis/scene.blueprint.administration test:browser - yarn workspace @cratis/scene.blueprint.project-tracker test:browser - yarn workspace @cratis/scene.blueprint.orders test:browser - yarn workspace @cratis/scene.blueprint.catalog test:browser - yarn workspace @cratis/scene.blueprint.vendors test:browser - yarn workspace @cratis/scene.blueprint.invoicing test:browser - yarn workspace @cratis/scene.blueprint.customer-portal test:browser - yarn workspace @cratis/scene.blueprint.contracts test:browser - yarn workspace @cratis/scene.blueprint.work-orders test:browser - yarn workspace @cratis/scene.blueprint.bookings test:browser + failed=() + for w in $WORKSPACES; do + yarn workspace "$w" test:browser || failed+=("$w") + done + if (( ${#failed[@]} )); then + printf 'Failed browser workspace: %s\n' "${failed[@]}" + exit 1 + fi + + - name: Upload specification reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: spec-reports-browser-${{ matrix.shard }} + path: spec-reports/ + retention-days: 3 + if-no-files-found: ignore + + - name: Summarize specification reports + if: always() + run: node .github/scripts/summarize-spec-reports.mjs spec-reports >> "$GITHUB_STEP_SUMMARY" From 10efd542b87b440f7c95f24bc1a02081c1f43036 Mon Sep 17 00:00:00 2001 From: woksin Date: Sun, 25 Oct 2026 05:50:16 +0100 Subject: [PATCH 3/4] Drop the path trigger for the removed browser global setup --- .github/workflows/javascript-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/javascript-build.yml b/.github/workflows/javascript-build.yml index edf1d862..50f4a503 100644 --- a/.github/workflows/javascript-build.yml +++ b/.github/workflows/javascript-build.yml @@ -25,7 +25,6 @@ on: - ".github/scripts/summarize-spec-reports.*" - ".github/scripts/browser-spec-shards.*" - "vite.base.ts" - - "vitest.browser.global-setup.ts" jobs: build: From e4e678689a13db88ca92831696097506b5635a58 Mon Sep 17 00:00:00 2001 From: woksin Date: Sun, 25 Oct 2026 13:02:24 +0100 Subject: [PATCH 4/4] Reject specification reports with no tests or invalid counts A zero-test JUnit suite or TRX counter, or a non-numeric count, used to summarize successfully and let a shard report green without evidence. Counts must now be non-negative integers, durations finite, and every report must record at least one test. --- .github/scripts/summarize-spec-reports.mjs | 27 +++++++++++---- .../scripts/summarize-spec-reports.test.mjs | 34 +++++++++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/.github/scripts/summarize-spec-reports.mjs b/.github/scripts/summarize-spec-reports.mjs index 5542fc2f..a3466591 100644 --- a/.github/scripts/summarize-spec-reports.mjs +++ b/.github/scripts/summarize-spec-reports.mjs @@ -20,6 +20,19 @@ function seconds(duration = '0') { return duration.split(':').reduce((total, part) => total * 60 + Number(part), 0); } +function count(value, attribute, name) { + const number = Number(value ?? 0); + if (!Number.isInteger(number) || number < 0) throw new Error(`Invalid ${attribute} "${value}" in ${name}`); + return number; +} + +function validated(report) { + const durations = [report.duration, ...report.slow.map(item => item.duration)]; + if (durations.some(duration => !Number.isFinite(duration) || duration < 0)) throw new Error(`Invalid duration in ${report.name}`); + if (report.tests === 0) throw new Error(`No tests recorded in ${report.name}`); + return report; +} + function cell(value) { return String(value).replace(/\|/g, '\\|').replace(/[\r\n]/g, ' '); } @@ -28,9 +41,9 @@ function junit(document, name) { const suites = document.testsuite; if (!suites.length) throw new Error(`No test suites found in ${name}`); const totals = suites.reduce((sum, suite) => ({ - tests: sum.tests + Number(suite.tests ?? 0), - failed: sum.failed + Number(suite.failures ?? 0) + Number(suite.errors ?? 0), - skipped: sum.skipped + Number(suite.skipped ?? 0), + tests: sum.tests + count(suite.tests, 'tests', name), + failed: sum.failed + count(suite.failures, 'failures', name) + count(suite.errors, 'errors', name), + skipped: sum.skipped + count(suite.skipped, 'skipped', name), duration: sum.duration + seconds(suite.time) }), { tests: 0, failed: 0, skipped: 0, duration: 0 }); return { name, ...totals, slow: suites.map(suite => ({ name: suite.name ?? name, duration: seconds(suite.time) })) }; @@ -44,9 +57,9 @@ function trx(document, name) { const slow = results.map(result => ({ name: result.testName ?? name, duration: seconds(result.duration) })); return { name, - tests: Number(counters.total ?? results.length), - failed: Number(counters.failed ?? 0) + Number(counters.error ?? 0) + Number(counters.timeout ?? 0) + Number(counters.aborted ?? 0), - skipped: Number(counters.notExecuted ?? 0), + tests: count(counters.total ?? results.length, 'total', name), + failed: count(counters.failed, 'failed', name) + count(counters.error, 'error', name) + count(counters.timeout, 'timeout', name) + count(counters.aborted, 'aborted', name), + skipped: count(counters.notExecuted, 'notExecuted', name), duration: times.start && times.finish ? (Date.parse(times.finish) - Date.parse(times.start)) / 1000 : slow.reduce((sum, test) => sum + test.duration, 0), slow }; @@ -64,7 +77,7 @@ if (!files.length) { const reports = files.map(file => { const path = join(directory, file); const document = readReport(path); - return file.endsWith('.trx') ? trx(document, relative(directory, path)) : junit(document, basename(file, '.junit.xml')); + return validated(file.endsWith('.trx') ? trx(document, relative(directory, path)) : junit(document, basename(file, '.junit.xml'))); }); console.log('## Specification results\n\n| Workspace / project | Tests | Failed | Skipped | Seconds |\n| --- | ---: | ---: | ---: | ---: |'); for (const report of reports) { diff --git a/.github/scripts/summarize-spec-reports.test.mjs b/.github/scripts/summarize-spec-reports.test.mjs index 2659080a..0568907f 100644 --- a/.github/scripts/summarize-spec-reports.test.mjs +++ b/.github/scripts/summarize-spec-reports.test.mjs @@ -89,3 +89,37 @@ test('refuses an empty report directory', t => { assert.equal(result.status, 2); assert.match(result.stderr, /No specification reports found/); }); + +test('refuses reports that record no tests', t => { + const directory = fixture(t); + for (const [file, xml] of [ + ['zero.junit.xml', ''], + ['zero.trx', ''] + ]) { + const report = join(directory, file); + writeFileSync(report, xml); + const result = summarize(directory); + assert.equal(result.status, 2); + assert.match(result.stderr, /No tests recorded/); + assert.equal(result.stdout, ''); + rmSync(report); + } +}); + +test('refuses non-numeric or negative counts and durations', t => { + const directory = fixture(t); + for (const [file, xml, error] of [ + ['count.junit.xml', '', /Invalid tests/], + ['negative.junit.xml', '', /Invalid failures/], + ['time.junit.xml', '', /Invalid duration/], + ['count.trx', '', /Invalid total/] + ]) { + const report = join(directory, file); + writeFileSync(report, xml); + const result = summarize(directory); + assert.equal(result.status, 2); + assert.match(result.stderr, error); + assert.equal(result.stdout, ''); + rmSync(report); + } +});