Publish specification timing reports and shard browser specs in CI

.github/scripts/browser-spec-shards.mjs

@@ -0,0 +1,49 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { readFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { globSync } from 'glob';
+
+// Local baseline wall-time medians (seconds); replace with CI measurements when available.
+const weights = {
+    '@cratis/scene.primereact': 31.37,
+    '@cratis/scene.blueprint.default': 43.24,
+    '@cratis/scene.blueprint.administration': 27.17,
+    '@cratis/scene.blueprint.crm': 94.65
+};
+
+function browserWorkspaces(root) {
+    const { workspaces } = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8'));
+    const ignore = ['**/node_modules/**', ...workspaces.filter(pattern => pattern.startsWith('!')).map(pattern => pattern.slice(1))];
+    const patterns = workspaces.filter(pattern => !pattern.startsWith('!')).map(pattern => `${pattern}/package.json`);
+    return globSync(patterns, { cwd: root, ignore, nodir: true })
+        .map(file => JSON.parse(readFileSync(join(root, file), 'utf8')))
+        .filter(manifest => manifest.scripts?.['test:browser'])
+        .map(manifest => ({ name: manifest.name, weight: weights[manifest.name] ?? 10 }))
+        .sort((left, right) => right.weight - left.weight || left.name.localeCompare(right.name));
+}
+
+export function browserSpecShards(root, count = 3) {
+    if (!Number.isInteger(count) || count < 1) throw new Error('Shard count must be a positive integer');
+    const workspaces = browserWorkspaces(root);
+    if (!workspaces.length) throw new Error('No browser specification workspaces found');
+    const shards = Array.from({ length: Math.min(count, workspaces.length) }, (_, index) => ({ shard: index + 1, workspaces: [], weight: 0 }));
+    for (const workspace of workspaces) {
+        const lightest = shards.reduce((best, shard) => shard.weight < best.weight ? shard : best);
+        lightest.workspaces.push(workspace.name);
+        lightest.weight += workspace.weight;
+    }
+    return shards.map(({ shard, workspaces }) => ({ shard, workspaces: workspaces.join(' ') }));
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+    try {
+        const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
+        console.log(JSON.stringify(browserSpecShards(root, Number(process.argv[2] ?? 3))));
+    } catch (error) {
+        console.error(error.message);
+        process.exitCode = 1;
+    }
+}

.github/scripts/browser-spec-shards.test.mjs

@@ -0,0 +1,94 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { test } from 'node:test';
+import { browserSpecShards } from './browser-spec-shards.mjs';
+
+const scripts = dirname(fileURLToPath(import.meta.url));
+const root = resolve(scripts, '../..');
+
+test('assigns every real browser workspace exactly once across at most three nonempty shards', () => {
+    const directory = join(root, 'Source/JavaScript');
+    const expected = readdirSync(directory).map(name => JSON.parse(readFileSync(join(directory, name, 'package.json'), 'utf8')))
+        .filter(manifest => manifest.scripts?.['test:browser']).map(manifest => manifest.name).sort();
+    const shards = browserSpecShards(root);
+    const actual = shards.flatMap(shard => shard.workspaces.split(' '));
+    assert.ok(expected.length > 0);
+    assert.ok(shards.length > 0 && shards.length <= 3);
+    assert.equal(new Set(actual).size, actual.length);
+    assert.deepEqual(actual.sort(), expected);
+    assert.deepEqual(shards.map(shard => shard.shard), [1, 2, 3]);
+});
+
+function fixture(t, count = 0) {
+    mkdirSync(join(root, '.ai-work/262'), { recursive: true });
+    const directory = mkdtempSync(join(root, '.ai-work/262/shards-'));
+    t.after(() => rmSync(directory, { recursive: true }));
+    writeFileSync(join(directory, 'package.json'), JSON.stringify({ workspaces: ['Source/JavaScript/**/*', '!**/dist/**', '!**/obj/**', '!**/bin/**'] }));
+    mkdirSync(join(directory, 'Source/JavaScript'), { recursive: true });
+    for (let index = 0; index < count; index++) {
+        const workspace = join(directory, `Source/JavaScript/workspace${index}`);
+        mkdirSync(workspace);
+        writeFileSync(join(workspace, 'package.json'), JSON.stringify({ name: `@test/workspace${index}`, scripts: { 'test:browser': 'vitest' } }));
+    }
+    return directory;
+}
+
+test('uses deterministic greedy balancing and does not create empty shards', t => {
+    const directory = fixture(t, 2);
+    assert.deepEqual(browserSpecShards(directory), [
+        { shard: 1, workspaces: '@test/workspace0' },
+        { shard: 2, workspaces: '@test/workspace1' }
+    ]);
+    assert.deepEqual(browserSpecShards(directory, 1), [{ shard: 1, workspaces: '@test/workspace0 @test/workspace1' }]);
+    assert.throws(() => browserSpecShards(directory, 0), /positive integer/);
+});
+
+test('discovers nested declared workspaces but not build output or installed dependencies', t => {
+    const directory = fixture(t, 1);
+    for (const name of ['nested/browser', 'nested/dist/cjs', 'nested/obj/cache', 'nested/bin/Debug', 'node_modules/dependency']) {
+        const workspace = join(directory, 'Source/JavaScript', name);
+        mkdirSync(workspace, { recursive: true });
+        writeFileSync(join(workspace, 'package.json'), JSON.stringify({ name, scripts: { 'test:browser': 'vitest' } }));
+    }
+    const actual = browserSpecShards(directory).flatMap(shard => shard.workspaces.split(' ')).sort();
+    assert.deepEqual(actual, ['@test/workspace0', 'nested/browser']);
+});
+
+test('CLI prints compact JSON for the discovered workspaces', () => {
+    const result = spawnSync(process.execPath, [join(scripts, 'browser-spec-shards.mjs'), '3'], { encoding: 'utf8' });
+    assert.equal(result.status, 0);
+    assert.equal(result.stdout.trim(), JSON.stringify(browserSpecShards(root)));
+});
+
+test('the workflow runs every workspace in a shard and preserves a failure', t => {
+    const directory = fixture(t);
+    const yarn = join(directory, 'yarn');
+    writeFileSync(yarn, '#!/bin/bash\nprintf "%s\\n" "$2" >> "$CALLS"\n[[ "$2" != "$FAILED_WORKSPACE" ]]\n', { mode: 0o755 });
+    const workflow = readFileSync(join(root, '.github/workflows/javascript-build.yml'), 'utf8');
+    const loop = workflow.match(/        run: \|\n(          failed=\(\)[\s\S]*?)\n\n/)[1].replace(/^          /gm, '');
+    for (const failed of ['', '@test/first']) {
+        const calls = join(directory, `calls-${failed ? 'failed' : 'passed'}`);
+        const result = spawnSync('bash', ['-e', '-c', loop], {
+            encoding: 'utf8',
+            env: { ...process.env, PATH: `${directory}:${process.env.PATH}`, CALLS: calls, FAILED_WORKSPACE: failed, WORKSPACES: '@test/first @test/second' }
+        });
+        assert.equal(result.status, failed ? 1 : 0, result.stderr);
+        assert.equal(readFileSync(calls, 'utf8'), '@test/first\n@test/second\n');
+    }
+});
+
+test('CLI fails rather than silently accepting an empty discovery', t => {
+    const directory = fixture(t);
+    mkdirSync(join(directory, '.github/scripts'), { recursive: true });
+    const script = join(directory, '.github/scripts/browser-spec-shards.mjs');
+    copyFileSync(join(scripts, 'browser-spec-shards.mjs'), script);
+    const result = spawnSync(process.execPath, [script], { encoding: 'utf8' });
+    assert.equal(result.status, 1);
+    assert.match(result.stderr, /No browser specification workspaces found/);
+});

.github/scripts/summarize-spec-reports.mjs

@@ -0,0 +1,93 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { spawnSync } from 'node:child_process';
+import { existsSync, readdirSync } from 'node:fs';
+import { basename, join, relative } from 'node:path';
+
+function readReport(path) {
+    // Python's XML parser is also required by the existing specification runner.
+    const parser = `import json, sys, xml.etree.ElementTree as ET
+root = ET.parse(sys.argv[1]).getroot()
+names = ('testsuite', 'Counters', 'UnitTestResult', 'Times')
+print(json.dumps({name: [e.attrib for e in root.iter() if e.tag.split('}')[-1] == name] for name in names}))`;
+    const result = spawnSync('python3', ['-c', parser, path], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+    if (result.error || result.status !== 0) throw new Error(`Cannot parse ${path}: ${result.error?.message ?? result.stderr.trim()}`);
+    return JSON.parse(result.stdout);
+}
+
+function seconds(duration = '0') {
+    return duration.split(':').reduce((total, part) => total * 60 + Number(part), 0);
+}
+
+function count(value, attribute, name) {
+    const number = Number(value ?? 0);
+    if (!Number.isInteger(number) || number < 0) throw new Error(`Invalid ${attribute} "${value}" in ${name}`);
+    return number;
+}
+
+function validated(report) {
+    const durations = [report.duration, ...report.slow.map(item => item.duration)];
+    if (durations.some(duration => !Number.isFinite(duration) || duration < 0)) throw new Error(`Invalid duration in ${report.name}`);
+    if (report.tests === 0) throw new Error(`No tests recorded in ${report.name}`);
+    return report;
+}
+
+function cell(value) {
+    return String(value).replace(/\|/g, '\\|').replace(/[\r\n]/g, ' ');
+}
+
+function junit(document, name) {
+    const suites = document.testsuite;
+    if (!suites.length) throw new Error(`No test suites found in ${name}`);
+    const totals = suites.reduce((sum, suite) => ({
+        tests: sum.tests + count(suite.tests, 'tests', name),
+        failed: sum.failed + count(suite.failures, 'failures', name) + count(suite.errors, 'errors', name),
+        skipped: sum.skipped + count(suite.skipped, 'skipped', name),
+        duration: sum.duration + seconds(suite.time)
+    }), { tests: 0, failed: 0, skipped: 0, duration: 0 });
+    return { name, ...totals, slow: suites.map(suite => ({ name: suite.name ?? name, duration: seconds(suite.time) })) };
+}
+
+function trx(document, name) {
+    if (document.Counters.length !== 1) throw new Error(`Expected one TRX counter in ${name}`);
+    const counters = document.Counters[0];
+    const results = document.UnitTestResult;
+    const times = document.Times[0] ?? {};
+    const slow = results.map(result => ({ name: result.testName ?? name, duration: seconds(result.duration) }));
+    return {
+        name,
+        tests: count(counters.total ?? results.length, 'total', name),
+        failed: count(counters.failed, 'failed', name) + count(counters.error, 'error', name) + count(counters.timeout, 'timeout', name) + count(counters.aborted, 'aborted', name),
+        skipped: count(counters.notExecuted, 'notExecuted', name),
+        duration: times.start && times.finish ? (Date.parse(times.finish) - Date.parse(times.start)) / 1000 : slow.reduce((sum, test) => sum + test.duration, 0),
+        slow
+    };
+}
+
+const directory = process.argv[2];
+const files = directory && existsSync(directory)
+    ? readdirSync(directory, { recursive: true }).filter(file => /(?:\.junit\.xml|\.trx)$/.test(file)).sort()
+    : [];
+
+if (!files.length) {
+    console.error(`No specification reports found in ${directory ?? '<missing directory>'}`);
+    process.exitCode = 2;
+} else try {
+    const reports = files.map(file => {
+        const path = join(directory, file);
+        const document = readReport(path);
+        return validated(file.endsWith('.trx') ? trx(document, relative(directory, path)) : junit(document, basename(file, '.junit.xml')));
+    });
+    console.log('## Specification results\n\n| Workspace / project | Tests | Failed | Skipped | Seconds |\n| --- | ---: | ---: | ---: | ---: |');
+    for (const report of reports) {
+        console.log(`| ${cell(report.name)} | ${report.tests} | ${report.failed} | ${report.skipped} | ${report.duration.toFixed(3)} |`);
+    }
+    console.log('\n### Slowest 10 files / tests\n\n| Report | File / test | Seconds |\n| --- | --- | ---: |');
+    const slowest = reports.flatMap(report => report.slow.map(item => ({ ...item, report: report.name })))
+        .sort((left, right) => right.duration - left.duration).slice(0, 10);
+    for (const item of slowest) console.log(`| ${cell(item.report)} | ${cell(item.name)} | ${item.duration.toFixed(3)} |`);
+} catch (error) {
+    console.error(error.message);
+    process.exitCode = 2;
+}

.github/scripts/summarize-spec-reports.test.mjs

@@ -0,0 +1,125 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { test } from 'node:test';
+
+const scripts = dirname(fileURLToPath(import.meta.url));
+const root = resolve(scripts, '../..');
+
+function fixture(t) {
+    mkdirSync(join(root, '.ai-work/262'), { recursive: true });
+    const directory = mkdtempSync(join(root, '.ai-work/262/summary-'));
+    t.after(() => rmSync(directory, { recursive: true }));
+    return directory;
+}
+
+function summarize(directory) {
+    const result = spawnSync(process.execPath, [join(scripts, 'summarize-spec-reports.mjs'), directory], { encoding: 'utf8' });
+    assert.ifError(result.error);
+    return result;
+}
+
+test('summarizes nested JUnit reports without counting the root twice, even on test failure', t => {
+    const directory = fixture(t);
+    mkdirSync(join(directory, 'nested'));
+    writeFileSync(join(directory, 'nested/model.junit.xml'), '<testsuites tests="3"><testsuite name="slow&amp;file|name" tests="2" failures="1" errors="0" skipped="1" time="2.5"></testsuite><testsuite name="fast" tests="1" time="0.5"></testsuite></testsuites>');
+    const result = summarize(directory);
+    assert.equal(result.status, 0);
+    assert.match(result.stdout, /\| model \| 3 \| 1 \| 1 \| 3\.000 \|/);
+    assert.match(result.stdout, /slow&file\\\|name \| 2\.500/);
+    assert.ok(result.stdout.indexOf('slow&file') < result.stdout.indexOf('| fast |'));
+});
+
+test('summarizes TRX counters, elapsed time and per-test duration', t => {
+    const directory = fixture(t);
+    writeFileSync(join(directory, 'Engine.Specs.trx'), '<TestRun><Times start="2026-10-24T12:00:00Z" finish="2026-10-24T12:01:03Z"/><Results><UnitTestResult testName="slow" duration="00:01:02.500" outcome="Failed"/><UnitTestResult testName="fast" duration="00:00:00.500" outcome="Passed"/></Results><ResultSummary><Counters total="3" failed="1" notExecuted="1"/></ResultSummary></TestRun>');
+    const result = summarize(directory);
+    assert.equal(result.status, 0);
+    assert.match(result.stdout, /Engine.Specs.trx \| 3 \| 1 \| 1 \| 63\.000/);
+    assert.match(result.stdout, /slow \| 62\.500/);
+});
+
+test('keeps the slowest ten entries across reports', t => {
+    const directory = fixture(t);
+    const suites = Array.from({ length: 12 }, (_, index) => `<testsuite name="file-${index}" tests="1" time="${index}"/>`).join('');
+    writeFileSync(join(directory, 'many.junit.xml'), `<testsuites>${suites}</testsuites>`);
+    const result = summarize(directory);
+    assert.equal(result.status, 0);
+    assert.match(result.stdout, /file-11 \| 11\.000/);
+    assert.doesNotMatch(result.stdout, /\| file-[01] \|/);
+});
+
+test('parses XML quoting and namespace prefixes rather than extracting attributes by regex', t => {
+    const directory = fixture(t);
+    writeFileSync(join(directory, 'quoted.junit.xml'), "<j:testsuites xmlns:j='urn:junit'><j:testsuite name='one &gt; zero' tests='1' time='0.5'/></j:testsuites>");
+    const result = summarize(directory);
+    assert.equal(result.status, 0);
+    assert.match(result.stdout, /quoted \| 1 \| 0 \| 0 \| 0\.500/);
+    assert.match(result.stdout, /one > zero/);
+});
+
+test('refuses incomplete or malformed XML instead of reporting partial results', t => {
+    const directory = fixture(t);
+    for (const xml of ['<testsuites><testsuite tests="1" time="1"/>', '<testsuites><testsuite></testsuites>', '<testsuites name="&invalid;"><testsuite/></testsuites>']) {
+        writeFileSync(join(directory, 'broken.junit.xml'), xml);
+        const result = summarize(directory);
+        assert.equal(result.status, 2);
+        assert.match(result.stderr, /Cannot parse/);
+        assert.equal(result.stdout, '');
+    }
+});
+
+test('refuses XML without the expected report structure', t => {
+    const directory = fixture(t);
+    for (const file of ['empty.junit.xml', 'empty.trx']) {
+        const report = join(directory, file);
+        writeFileSync(report, '<empty/>');
+        assert.equal(summarize(directory).status, 2);
+        rmSync(report);
+    }
+});
+
+test('refuses an empty report directory', t => {
+    const result = summarize(fixture(t));
+    assert.equal(result.status, 2);
+    assert.match(result.stderr, /No specification reports found/);
+});
+
+test('refuses reports that record no tests', t => {
+    const directory = fixture(t);
+    for (const [file, xml] of [
+        ['zero.junit.xml', '<testsuites><testsuite name="none" tests="0" time="0"/></testsuites>'],
+        ['zero.trx', '<TestRun><Results/><ResultSummary><Counters total="0"/></ResultSummary></TestRun>']
+    ]) {
+        const report = join(directory, file);
+        writeFileSync(report, xml);
+        const result = summarize(directory);
+        assert.equal(result.status, 2);
+        assert.match(result.stderr, /No tests recorded/);
+        assert.equal(result.stdout, '');
+        rmSync(report);
+    }
+});
+
+test('refuses non-numeric or negative counts and durations', t => {
+    const directory = fixture(t);
+    for (const [file, xml, error] of [
+        ['count.junit.xml', '<testsuites><testsuite name="bad" tests="invalid" time="1"/></testsuites>', /Invalid tests/],
+        ['negative.junit.xml', '<testsuites><testsuite name="bad" tests="1" failures="-1" time="1"/></testsuites>', /Invalid failures/],
+        ['time.junit.xml', '<testsuites><testsuite name="bad" tests="1" time="soon"/></testsuites>', /Invalid duration/],
+        ['count.trx', '<TestRun><Results/><ResultSummary><Counters total="many"/></ResultSummary></TestRun>', /Invalid total/]
+    ]) {
+        const report = join(directory, file);
+        writeFileSync(report, xml);
+        const result = summarize(directory);
+        assert.equal(result.status, 2);
+        assert.match(result.stderr, error);
+        assert.equal(result.stdout, '');
+        rmSync(report);
+    }
+});

.github/workflows/dotnet-build.yml

@@ -28,6 +28,8 @@ on:
       - "Source/DotNET/**"
       - ".github/scripts/run-specifications.sh"
       - ".github/scripts/run-specifications.test.mjs"
+      - ".github/scripts/summarize-spec-reports.mjs"
+      - ".github/scripts/summarize-spec-reports.test.mjs"
       - "Directory.Build.props"
       - "Directory.Packages.props"
       - "global.json"
@@ -134,10 +136,15 @@ jobs:
       - name: Run specification project (${{ matrix.project }})
         run: bash .github/scripts/run-specifications.sh Release ${{ matrix.project }}
 
-      - name: Upload failed specification results
-        if: failure()
+      - name: Upload specification results
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: specification-results-${{ strategy.job-index }}
           path: TestResults/
+          retention-days: 3
           if-no-files-found: ignore
+
+      - name: Summarize specification results
+        if: always()
+        run: node .github/scripts/summarize-spec-reports.mjs TestResults >> "$GITHUB_STEP_SUMMARY"