diff --git a/.github/scripts/benchmark-comment.cjs b/.github/scripts/pr-comment.cjs similarity index 55% rename from .github/scripts/benchmark-comment.cjs rename to .github/scripts/pr-comment.cjs index 24c3a3f..0baae77 100644 --- a/.github/scripts/benchmark-comment.cjs +++ b/.github/scripts/pr-comment.cjs @@ -1,9 +1,9 @@ -// Creates or updates the benchmark comment on a PR. Used by actions/github-script -// from the `benchmark` workflow (PRs from this repository) and from the -// `benchmark comment` workflow (PRs from forks and dependabot). +// Creates or updates a report comment on a PR (one comment per marker, kept up +// to date). Used by actions/github-script from the `benchmark` workflow (PRs +// from this repository) and from the `benchmark comment` workflow (PRs from +// forks and dependabot). const fs = require('node:fs'); -const MARKER = ''; const MAX_BODY = 65536; /** @@ -11,29 +11,44 @@ const MAX_BODY = 65536; * @param {import('@octokit/rest').Octokit} options.github * @param {object} options.context * @param {object} options.core + * @param {string} options.marker HTML comment that identifies the comment + * @param {string} options.title heading of the failure message * @param {number} options.pr PR number * @param {string} options.sha head commit of the PR - * @param {string} options.runUrl URL of the benchmark run - * @param {boolean} options.success whether the benchmark run succeeded + * @param {string} options.runUrl URL of the workflow run + * @param {boolean} options.success whether the job that wrote the report succeeded * @param {string} [options.reportPath] Markdown report (required on success) + * @param {boolean} [options.create] false: only update an existing comment */ -module.exports = async ({ github, context, core, pr, sha, runUrl, success, reportPath }) => { +module.exports = async ({ + github, + context, + core, + marker, + title, + pr, + sha, + runUrl, + success, + reportPath, + create = true, +}) => { const { owner, repo } = context.repo; const short = sha.slice(0, 7); let report = success ? fs.readFileSync(reportPath, 'utf8') : [ - '## ⏱️ Benchmark results', + `## ${title}`, '', '> [!CAUTION]', - `> The benchmark failed for \`${short}\`. See the [workflow run](${runUrl}).`, + `> The check failed for \`${short}\`. See the [workflow run](${runUrl}).`, ].join('\n'); const footer = `\n\n[Workflow run](${runUrl}) · updated for \`${short}\``; - const limit = MAX_BODY - MARKER.length - footer.length - 1; + const limit = MAX_BODY - marker.length - footer.length - 1; if (report.length > limit) report = report.slice(0, limit - 20) + '\n\n…(truncated)'; - const body = `${MARKER}\n${report}${footer}`; + const body = `${marker}\n${report}${footer}`; const comments = await github.paginate(github.rest.issues.listComments, { owner, @@ -42,14 +57,16 @@ module.exports = async ({ github, context, core, pr, sha, runUrl, success, repor per_page: 100, }); const existing = comments.find( - (c) => c.user?.login === 'github-actions[bot]' && c.body?.startsWith(MARKER), + (c) => c.user?.login === 'github-actions[bot]' && c.body?.startsWith(marker), ); if (existing) { await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); core.info(`Updated comment ${existing.html_url}`); - } else { + } else if (create) { const { data } = await github.rest.issues.createComment({ owner, repo, issue_number: pr, body }); core.info(`Created comment ${data.html_url}`); + } else { + core.info('Nothing to report and no comment to update, skipping.'); } }; diff --git a/.github/workflows/benchmark-comment.yml b/.github/workflows/benchmark-comment.yml index 3d8a8ed..16227fe 100644 --- a/.github/workflows/benchmark-comment.yml +++ b/.github/workflows/benchmark-comment.yml @@ -1,10 +1,11 @@ name: benchmark comment -# Posts the report of the `benchmark` workflow on PRs from forks and dependabot, -# which get a read-only token in `pull_request` (PRs from this repository are -# handled by the `comment` job of `benchmark`). It runs from the default -# branch with a write token, so it never checks out or runs code from the PR: -# the artifact is only text, and the PR is resolved from the run metadata. +# Posts the reports of the `benchmark` workflow (benchmarks and public API) on +# PRs from forks and dependabot, which get a read-only token in `pull_request` +# (PRs from this repository are handled by the `comment` and `api_comment` +# jobs of `benchmark`). It runs from the default branch with a write token, so +# it never checks out or runs code from the PR: the artifacts are only text +# and JSON data, and the PR is resolved from the run metadata. on: workflow_run: @@ -44,9 +45,22 @@ jobs: run-id: ${{ github.event.workflow_run.id }} github-token: ${{ github.token }} + # the api job can succeed when a benchmark fails: no artifact when it failed + - uses: actions/download-artifact@v8 + id: api + continue-on-error: true + with: + name: api + path: ${{ runner.temp }}/api + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - uses: actions/github-script@v9 + env: + API: ${{ steps.api.outcome == 'success' }} with: script: | + const fs = require('node:fs'); const run = context.payload.workflow_run; const { owner, repo } = context.repo; @@ -67,13 +81,27 @@ jobs: return; } - await require('./.github/scripts/benchmark-comment.cjs')({ - github, - context, - core, - pr, - sha: run.head_sha, - runUrl: run.html_url, + const comment = require('./.github/scripts/pr-comment.cjs'); + const common = { github, context, core, pr, sha: run.head_sha, runUrl: run.html_url }; + + await comment({ + ...common, + marker: '', + title: '⏱️ Benchmark results', success: run.conclusion === 'success', reportPath: `${process.env.RUNNER_TEMP}/benchmark/report.md`, }); + + // only a boolean is read from the JSON of the PR run + if (process.env.API === 'true') { + const dir = `${process.env.RUNNER_TEMP}/api`; + const { changed } = JSON.parse(fs.readFileSync(`${dir}/api.json`, 'utf8')); + await comment({ + ...common, + marker: '', + title: '🧩 Public API changes', + success: true, + reportPath: `${dir}/report.md`, + create: changed === true, + }); + } diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 3e9db00..ed651bc 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,14 +1,19 @@ name: benchmark -# Runs the benchmarks of the PR against the latest release on npm and posts -# the report as a PR comment (one comment, kept up to date). PRs from forks and -# dependabot get a read-only token, so the `benchmark comment` workflow posts -# for them. +# Compares the PR against the latest release on npm and posts two PR comments +# (each one kept up to date): +# - the benchmarks of every runtime; +# - the changes in the public API (type declarations and package.json entry +# points). Posted only when the API changes, or to update an earlier one. +# PRs from forks and dependabot get a read-only token, so the +# `benchmark comment` workflow posts for them. on: pull_request: branches: - master + # every input of the build, so a PR that skips the workflow cannot change + # the API either paths: - "src/**" - "Cargo.toml" @@ -16,11 +21,15 @@ on: - "package.json" - "pnpm-lock.yaml" - "bin/build.sh" + - "bin/wasm-opt.sh" + - "bin/install-binaryen.sh" - "js/**" + - "tsconfig.json" - "tsdown.config.ts" - "bin/bench-*.mjs" + - "bin/api-diff.mjs" - "test/browser/**" - - ".github/scripts/benchmark-comment.cjs" + - ".github/scripts/pr-comment.cjs" - ".github/workflows/benchmark.yml" concurrency: @@ -31,26 +40,25 @@ permissions: contents: read jobs: - benchmark: + # builds head once and downloads base, so every runtime measures the same + # two builds + build: runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 15 - env: - HEAD_SHA: ${{ github.event.pull_request.head.sha }} + outputs: + latest: ${{ steps.latest.outputs.version }} steps: - uses: actions/checkout@v7 - # same paths and key as the release workflow, so the cache of master is reused - - uses: actions/cache@v6 + # same shared key as the build of the release workflow, so the cache of + # master is reused. Only the release workflow saves it: two jobs that + # save the same key at the same time fail to reserve it. + - uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- + shared-key: release + save-if: false # wasm-bindgen-cli must match the wasm-bindgen crate in Cargo.lock - name: reading wasm-bindgen version @@ -75,51 +83,252 @@ jobs: # the base is the published package: inside the repository, because the # browser benchmark only serves files from there - name: downloading latest + id: latest run: | meta=$(curl -fsSL https://registry.npmjs.org/rregex/latest) curl -fsSL -o "$RUNNER_TEMP/rregex.tgz" "$(jq -r .dist.tarball <<< "$meta")" test "sha512-$(openssl dgst -sha512 -binary "$RUNNER_TEMP/rregex.tgz" | base64 -w0)" = "$(jq -r .dist.integrity <<< "$meta")" mkdir -p .latest - tar xzf "$RUNNER_TEMP/rregex.tgz" -C .latest --strip-components=1 package/lib - test -f .latest/lib/esm.mjs -a -f .latest/lib/rregex.wasm - echo "LATEST_VERSION=$(jq -r .version <<< "$meta")" >> "$GITHUB_ENV" + tar xzf "$RUNNER_TEMP/rregex.tgz" -C .latest --strip-components=1 package/lib package/package.json + test -f .latest/lib/esm.mjs -a -f .latest/lib/rregex.wasm -a -f .latest/package.json + echo "version=$(jq -r .version <<< "$meta")" >> "$GITHUB_OUTPUT" + + - name: installing + run: pnpm install --frozen-lockfile + + # native binaryen for bin/wasm-opt.sh; rust-cache does not keep it + - name: reading binaryen version + id: binaryen + run: echo "version=$(node -p 'require("./node_modules/binaryen/package.json").version')" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v6 + with: + path: target/binaryen + key: ${{ runner.os }}-binaryen-${{ steps.binaryen.outputs.version }} - name: setup - run: | - pnpm install --frozen-lockfile - node --run setup + run: node --run setup - # check that the lib is complete + # check that the lib is complete; the build time goes to the report - name: building head run: | - node --run build + BUILD_TIMINGS=build.json node --run build test -f lib/esm.mjs -a -f lib/rregex.wasm - # Chrome and Firefox come with the runner image; the runtimes run one - # after the other, so they do not compete for the CPU + - uses: actions/upload-artifact@v7 + with: + name: bench-libs + path: | + lib/ + .latest/lib/ + .latest/package.json + build.json + include-hidden-files: true # .latest + if-no-files-found: error + retention-days: 1 + + # One job for each runtime, each on its own runner, so they run in parallel + # and do not compete for the CPU. Every delta compares base and head on the + # same runner, so the runtimes do not need the same hardware. + bench: + needs: build + + runs-on: ubuntu-24.04 + timeout-minutes: 20 + + strategy: + matrix: + runtime: [node, chrome, firefox] + + steps: + - uses: actions/checkout@v7 + + # tinybench, for the Node benchmark only + - uses: pnpm/action-setup@v6 + if: matrix.runtime == 'node' + with: + version: 11 + + # also the runtime of the Node benchmark + - name: Run actions/setup-node@v7 + uses: actions/setup-node@v7 + with: + node-version: "26" + cache: ${{ matrix.runtime == 'node' && 'pnpm' || '' }} + + - uses: actions/download-artifact@v8 + with: + name: bench-libs + + - name: installing + if: matrix.runtime == 'node' + run: pnpm install --frozen-lockfile + + # Chrome and Firefox come with the runner image - name: running benchmarks env: - REPORT_DIR: ${{ runner.temp }}/benchmark + RUNTIME: ${{ matrix.runtime }} + run: | + mkdir -p "$RUNNER_TEMP/benchmark" + if [ "$RUNTIME" = node ]; then + node bin/bench-node.mjs --base .latest/lib --head lib \ + --json "$RUNNER_TEMP/benchmark/$RUNTIME.json" + else + node bin/bench-browser.mjs --browser "$RUNTIME" --base .latest/lib --head lib \ + --json "$RUNNER_TEMP/benchmark/$RUNTIME.json" + fi + + - uses: actions/upload-artifact@v7 + with: + name: benchmark-${{ matrix.runtime }} + path: ${{ runner.temp }}/benchmark/${{ matrix.runtime }}.json + if-no-files-found: error + retention-days: 1 + + # the public API of head vs the latest release, next to the benchmarks + api: + needs: build + + runs-on: ubuntu-24.04 + timeout-minutes: 5 + + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + LATEST_VERSION: ${{ needs.build.outputs.latest }} + + steps: + # api-diff.mjs only needs bin/, package.json, Node and git + - uses: actions/checkout@v7 + with: + sparse-checkout: bin + + - uses: actions/setup-node@v7 + with: + node-version: "26" + + - uses: actions/download-artifact@v8 + with: + name: bench-libs + + - name: comparing the public API + env: + REPORT_DIR: ${{ runner.temp }}/api run: | mkdir -p "$REPORT_DIR" - for browser in chrome firefox; do - node bin/bench-browser.mjs \ - --browser "$browser" \ - --base .latest/lib \ - --head lib \ - --json "$REPORT_DIR/$browser.json" - done + node bin/api-diff.mjs \ + --base .latest/lib \ + --head lib \ + --base-package .latest/package.json \ + --head-package package.json \ + --base-label "\`rregex@${LATEST_VERSION}\` (npm latest)" \ + --head-label "\`${HEAD_SHA::7}\`" \ + --markdown "$REPORT_DIR/report.md" \ + --json "$REPORT_DIR/api.json" + cat "$REPORT_DIR/report.md" >> "$GITHUB_STEP_SUMMARY" + + # the `benchmark comment` workflow downloads it by this name + - uses: actions/upload-artifact@v7 + with: + name: api + path: ${{ runner.temp }}/api + retention-days: 7 + + api_comment: + # PRs from this repository only, see `benchmark comment` for the others + if: >- + needs.api.result == 'success' && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login != 'dependabot[bot]' + + needs: api + + runs-on: ubuntu-24.04 + + permissions: + contents: read + pull-requests: write # to create and update the comment + + steps: + - uses: actions/checkout@v7 + with: + sparse-checkout: .github/scripts + + - uses: actions/download-artifact@v8 + with: + name: api + path: ${{ runner.temp }}/api + + # a new comment only when the API changes; an earlier one is updated + # either way, so it does not show changes that the PR no longer has + - uses: actions/github-script@v9 + with: + script: | + const fs = require('node:fs'); + const dir = `${process.env.RUNNER_TEMP}/api`; + const { changed } = JSON.parse(fs.readFileSync(`${dir}/api.json`, 'utf8')); + await require('./.github/scripts/pr-comment.cjs')({ + github, + context, + core, + marker: '', + title: '🧩 Public API changes', + pr: context.payload.pull_request.number, + sha: context.payload.pull_request.head.sha, + runUrl: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + success: true, + reportPath: `${dir}/report.md`, + create: changed === true, + }); + + report: + needs: [build, bench] + + runs-on: ubuntu-24.04 + timeout-minutes: 5 + + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + LATEST_VERSION: ${{ needs.build.outputs.latest }} + + steps: + # bench-compare.mjs only needs bin/ and Node + - uses: actions/checkout@v7 + with: + sparse-checkout: bin + + - uses: actions/setup-node@v7 + with: + node-version: "26" + + - uses: actions/download-artifact@v8 + with: + name: bench-libs + + - uses: actions/download-artifact@v8 + with: + pattern: benchmark-* + merge-multiple: true + path: ${{ runner.temp }}/benchmark + + - name: writing report + env: + REPORT_DIR: ${{ runner.temp }}/benchmark + run: | + cp build.json "$REPORT_DIR/build.json" node bin/bench-compare.mjs \ --base .latest/lib \ --head lib \ --base-label "\`rregex@${LATEST_VERSION}\` (npm latest)" \ --head-label "\`${HEAD_SHA::7}\`" \ + --runtime "$REPORT_DIR/node.json" \ --runtime "$REPORT_DIR/chrome.json" \ --runtime "$REPORT_DIR/firefox.json" \ + --build-time "$REPORT_DIR/build.json" \ --markdown "$REPORT_DIR/report.md" \ --json "$REPORT_DIR/results.json" cat "$REPORT_DIR/report.md" >> "$GITHUB_STEP_SUMMARY" + # the `benchmark comment` workflow downloads it by this name - uses: actions/upload-artifact@v7 with: name: benchmark @@ -129,11 +338,13 @@ jobs: comment: # PRs from this repository only, see `benchmark comment` for the others if: >- - !cancelled() && needs.benchmark.result != 'skipped' && + !cancelled() && needs.build.result != 'skipped' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login != 'dependabot[bot]' - needs: benchmark + # all of them: when build or bench fails, report is skipped and the + # comment reports the failure + needs: [build, bench, report] runs-on: ubuntu-24.04 @@ -147,20 +358,22 @@ jobs: sparse-checkout: .github/scripts - uses: actions/download-artifact@v8 - if: needs.benchmark.result == 'success' + if: needs.report.result == 'success' with: name: benchmark path: ${{ runner.temp }}/benchmark - uses: actions/github-script@v9 env: - SUCCESS: ${{ needs.benchmark.result == 'success' }} + SUCCESS: ${{ needs.report.result == 'success' }} with: script: | - await require('./.github/scripts/benchmark-comment.cjs')({ + await require('./.github/scripts/pr-comment.cjs')({ github, context, core, + marker: '', + title: '⏱️ Benchmark results', pr: context.payload.pull_request.number, sha: context.payload.pull_request.head.sha, runUrl: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 06fa6d7..b551243 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,15 +57,12 @@ jobs: registry-url: https://registry.npmjs.org/ cache: pnpm - - uses: actions/cache@v6 + # cargo registry and dependencies in target/. The key includes the rustc + # version, so a new Rust on the runner does not restore a stale cache. + # The benchmark workflow uses the same shared key. + - uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- + shared-key: release # wasm-bindgen-cli must match the wasm-bindgen crate in Cargo.lock - name: reading wasm-bindgen version @@ -77,18 +74,22 @@ jobs: with: tool: wasm-pack,wasm-bindgen@${{ steps.wasm-bindgen.outputs.version }} - - name: checking regex engine versions - run: ./bin/check-regex-deps.sh - - name: installing run: pnpm install --frozen-lockfile + # native binaryen for bin/wasm-opt.sh; rust-cache does not keep it + - name: reading binaryen version + id: binaryen + run: echo "version=$(node -p 'require("./node_modules/binaryen/package.json").version')" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v6 + with: + path: target/binaryen + key: ${{ runner.os }}-binaryen-${{ steps.binaryen.outputs.version }} + - name: setup run: node --run setup - - name: testing wasm - run: node --run test:rs -- --node - - name: building run: node --run build @@ -97,89 +98,176 @@ jobs: name: lib path: lib/ - test_node: + # asks crates.io for the publish dates: in its own job, next to the build + check_deps: runs-on: ubuntu-24.04 - needs: build + steps: + - uses: actions/checkout@v7 + with: + sparse-checkout: | + Cargo.lock + bin/check-regex-deps.sh + sparse-checkout-cone-mode: false - strategy: - matrix: - version: ["22", "24", "26"] + - name: checking regex engine versions + run: ./bin/check-regex-deps.sh + + # the Rust tests build the crate in the test profile: in their own job, so + # the build and the jobs that need `lib` do not wait for them + test_rs: + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v7 - - name: Run actions/setup-node@v7 (${{ matrix.version }}) - uses: actions/setup-node@v7 + - uses: actions/setup-node@v7 with: - node-version: ${{ matrix.version }} - registry-url: https://registry.npmjs.org/ + node-version: "26" + + - uses: Swatinem/rust-cache@v2 + + # wasm-bindgen-cli must match the wasm-bindgen crate in Cargo.lock + - name: reading wasm-bindgen version + id: wasm-bindgen + run: echo "version=$(grep -A1 '^name = "wasm-bindgen"$' Cargo.lock | sed -n 's/^version = "\(.*\)"$/\1/p')" >> "$GITHUB_OUTPUT" + + - uses: taiki-e/install-action@v2 + with: + tool: wasm-pack,wasm-bindgen@${{ steps.wasm-bindgen.outputs.version }} + + - run: rustup target add wasm32-unknown-unknown + + - name: testing wasm + run: node --run test:rs -- --node + + # Each runtime runs every version in one job: the tests take seconds, less + # than the setup of a runner, and one job per version used ~11 of the + # concurrent jobs of the account on each PR. A failed version stops the job, + # like the fail-fast of a matrix. + test_node: + runs-on: ubuntu-24.04 + + needs: build + + steps: + - uses: actions/checkout@v7 - uses: actions/download-artifact@v8 with: name: lib path: lib - - run: node lib/commonjs.cjs - - run: node lib/esm.mjs - - run: node --test test/node.test.cjs - - run: node --test test/node.test.mjs + - uses: actions/setup-node@v7 + with: + node-version: "22" + + - name: node 22 + run: &test-node | + node --version + node lib/commonjs.cjs + node lib/esm.mjs + node --test test/node.test.cjs + node --test test/node.test.mjs + + - uses: actions/setup-node@v7 + with: + node-version: "24" + + - name: node 24 + run: *test-node + + - uses: actions/setup-node@v7 + with: + node-version: "26" + + - name: node 26 + run: *test-node test_bun: runs-on: ubuntu-24.04 needs: build - strategy: - matrix: - version: ["1.0", "1.1", "1.2", "1.3", "1.4"] - steps: - uses: actions/checkout@v7 - - name: Run oven-sh/setup-bun@v2 (${{ matrix.version }}) - uses: oven-sh/setup-bun@v2 - with: - bun-version: ${{ matrix.version }} - - uses: actions/download-artifact@v8 with: name: lib path: lib - - run: bun lib/commonjs.cjs - - run: bun lib/esm.mjs - - run: bun test test/bun.test.cjs - - run: bun test test/bun.test.mjs + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.0" + + - name: bun 1.0 + run: &test-bun | + bun --version + bun lib/commonjs.cjs + bun lib/esm.mjs + bun test test/bun.test.cjs + bun test test/bun.test.mjs + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.1" + + - name: bun 1.1 + run: *test-bun + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.2" + + - name: bun 1.2 + run: *test-bun + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3" + + - name: bun 1.3 + run: *test-bun + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.4" + + - name: bun 1.4 + run: *test-bun test_deno: runs-on: ubuntu-24.04 needs: build - strategy: - matrix: - version: ["1.x", "2.x"] - steps: - uses: actions/checkout@v7 - - name: Run denoland/setup-deno@v2 (${{ matrix.version }}) - uses: denoland/setup-deno@v2 - with: - deno-version: "v${{ matrix.version }}" - - uses: actions/download-artifact@v8 with: name: lib path: lib - - run: deno install - if: ${{ matrix.version != '1.x' }} + - uses: denoland/setup-deno@v2 + with: + deno-version: v1.x + + - name: deno 1.x + run: &test-deno | + deno --version + deno run --allow-read=lib/rregex.wasm lib/esm.mjs + deno test --allow-read=lib/rregex.wasm test/deno.test.mjs - - run: deno run --allow-read=lib/rregex.wasm lib/esm.mjs + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + + - run: deno install - - run: deno test --allow-read=lib/rregex.wasm test/deno.test.mjs + - name: deno 2.x + run: *test-deno test_cf: runs-on: ubuntu-24.04 @@ -233,6 +321,8 @@ jobs: id-token: write # to enable use of OIDC for npm provenance needs: + - check_deps + - test_rs - test_node - test_bun - test_deno diff --git a/README.md b/README.md index 785c544..608224e 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,14 @@ The time of `rregex` is predictable: the time to copy the input plus a time that ### Run the benchmarks - **Browser:** run `pnpm build`, then `pnpm test:browser`, and open . The page shows the charts above and gives a Markdown table and SVG/PNG images of the results. -- **Node:** the same cases are in [`bin/bench-cases.mjs`](./bin/bench-cases.mjs). [`bin/bench-compare.mjs`](./bin/bench-compare.mjs) compares two builds of `rregex` with each other, and CI uses it on each pull request. +- **Node:** the same cases are in [`bin/bench-cases.mjs`](./bin/bench-cases.mjs). +- **Compare two builds:** [`bin/bench-node.mjs`](./bin/bench-node.mjs) and [`bin/bench-browser.mjs`](./bin/bench-browser.mjs) run base and head side by side in one runtime and write the samples to a JSON file. [`bin/bench-compare.mjs`](./bin/bench-compare.mjs) makes the report from these files. CI does this on each pull request, with one job for each runtime: + + ```sh + node bin/bench-node.mjs --base --head lib --json node.json + node bin/bench-browser.mjs --browser chrome --base --head lib --json chrome.json + node bin/bench-compare.mjs --base --head lib --runtime node.json --runtime chrome.json + ``` ## Known Issues diff --git a/bin/api-diff.mjs b/bin/api-diff.mjs new file mode 100755 index 0000000..5b05e06 --- /dev/null +++ b/bin/api-diff.mjs @@ -0,0 +1,187 @@ +#!/usr/bin/env node +// Compares the public API of two rregex builds: the type declarations in +// lib/ (*.d.ts, *.d.mts, *.d.cts) and the entry points of package.json +// (`exports`, `main`, `module`, `types`, `browser`). Writes a Markdown report +// with one diff for each change; files with the same change share one diff. +// +// Usage: +// node bin/api-diff.mjs --base --head \ +// [--base-package ] [--head-package ] \ +// [--base-label ] [--head-label ] \ +// [--markdown ] [--json ] +// +// The JSON file has `changed: true` when the API changed. Needs `git`. + +import { spawnSync } from 'node:child_process'; +import { mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { parseArgs } from 'node:util'; + +const { values: args } = parseArgs({ + options: { + base: { type: 'string' }, + head: { type: 'string' }, + 'base-package': { type: 'string' }, + 'head-package': { type: 'string' }, + 'base-label': { type: 'string', default: 'base' }, + 'head-label': { type: 'string', default: 'head' }, + markdown: { type: 'string' }, + json: { type: 'string' }, + }, +}); + +if (!args.base || !args.head) { + console.error('Usage: api-diff.mjs --base --head [options]'); + process.exit(2); +} + +// the comment limit of GitHub is 65536 chars, keep room for the footer +const MAX_MARKDOWN = 60000; +const ENTRY_FIELDS = ['main', 'module', 'types', 'browser', 'exports']; +const DTS = /\.d\.[cm]?ts$/; + +/** Unified diff of two files (null: the file does not exist), without the headers. */ +function diff(base, head) { + const { status, stdout, stderr } = spawnSync( + 'git', + ['diff', '--no-index', '--no-color', '--unified=3', '--', base ?? '/dev/null', head ?? '/dev/null'], + { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, + ); + // 0: same content, 1: different, anything else: git failed + if (status !== 0 && status !== 1) throw new Error(`git diff failed (${status}): ${stderr}`); + const lines = stdout.split('\n'); + const start = lines.findIndex((line) => line.startsWith('@@')); + return start === -1 ? '' : lines.slice(start).join('\n').trimEnd(); +} + +const listDts = async (dir) => (await readdir(dir)).filter((name) => DTS.test(name)); + +const dirs = { base: resolve(args.base), head: resolve(args.head) }; +const names = { + base: new Set(await listDts(dirs.base)), + head: new Set(await listDts(dirs.head)), +}; +const all = [...new Set([...names.base, ...names.head])].sort(); + +/** + * Changes, grouped by identical diff: { files, status, diff }. The line + * numbers of the hunks do not count, as the entry points have the same + * declarations at different lines. A removed file keeps no diff: its content + * is the base, not a change to review. + */ +const groups = new Map(); +const add = (file, status, body) => { + const key = `${status}\n${body.replace(/^@@ [^@]+ @@/gm, '@@')}`; + if (!groups.has(key)) groups.set(key, { files: [], status, diff: body }); + groups.get(key).files.push(file); +}; + +for (const name of all) { + const base = names.base.has(name) ? join(dirs.base, name) : null; + const head = names.head.has(name) ? join(dirs.head, name) : null; + if (!head) { + add(name, 'removed', ''); + continue; + } + const body = diff(base, head); + if (body) add(name, base ? 'modified' : 'added', body); +} + +// the entry points of package.json, as formatted JSON +if (args['base-package'] && args['head-package']) { + const tmp = await mkdtemp(join(tmpdir(), 'rregex-api-')); + try { + const entries = async (file) => { + const pkg = JSON.parse(await readFile(file, 'utf8')); + return JSON.stringify(Object.fromEntries(ENTRY_FIELDS.map((key) => [key, pkg[key]])), null, 2) + '\n'; + }; + const base = join(tmp, 'base.json'); + const head = join(tmp, 'head.json'); + await writeFile(base, await entries(args['base-package'])); + await writeFile(head, await entries(args['head-package'])); + const body = diff(base, head); + if (body) add('package.json', 'modified', body); + } finally { + await rm(tmp, { recursive: true, force: true }); + } +} + +const changes = [...groups.values()]; + +// ----- report ----- + +/** A code fence longer than any run of backticks in the text (JSDoc has ``` examples). */ +const fence = (text) => '`'.repeat(Math.max(3, ...[...text.matchAll(/`+/g)].map(([m]) => m.length + 1))); +const count = (body, sign) => body.split('\n').filter((line) => line.startsWith(sign)).length; +const files = (list) => list.map((file) => `\`${file}\``).join(', '); +const verb = { added: 'New file', removed: 'Removed file', modified: 'Changed' }; + +const lines = ['## 🧩 Public API changes', '']; +lines.push( + `Comparing the type declarations and the \`package.json\` entry points of **head** ${args['head-label']} ` + + `against **base** ${args['base-label']}.`, + '', +); + +if (!changes.length) { + lines.push('> [!NOTE]', '> No changes in the public API.'); +} else { + const total = changes.reduce((sum, change) => sum + change.files.length, 0); + lines.push( + '> [!IMPORTANT]', + `> The public API changes in **${total} file(s)**. In the diffs, lines that start with ` + + '`+` are added and lines that start with `-` are removed.', + '', + ); + + const truncated = []; + for (const change of changes) { + if (change.status === 'removed') { + lines.push(`### ${verb.removed}: ${files(change.files)}`, ''); + continue; + } + const added = count(change.diff, '+'); + const removed = count(change.diff, '-'); + const ticks = fence(change.diff); + const block = [ + `### ${verb[change.status]}: ${files(change.files)}`, + '', + `${added} line(s) added (+), ${removed} line(s) removed (-)` + + (change.files.length > 1 ? `; line numbers of \`${change.files[0]}\`` : '') + + '', + '', + `${ticks}diff`, + change.diff, + ticks, + '', + ]; + const size = block.reduce((sum, line) => sum + line.length + 1, 0); + if (lines.join('\n').length + size > MAX_MARKDOWN) { + truncated.push(...change.files); + continue; + } + lines.push(...block); + } + if (truncated.length) { + lines.push( + '> [!WARNING]', + `> The diffs of ${files(truncated)} are too long for this comment. ` + + 'The `api.json` file of the `api` artifact of the workflow run has all the diffs.', + '', + ); + } +} + +const markdown = lines.join('\n').trimEnd() + '\n'; +const json = { changed: changes.length > 0, changes }; + +if (args.markdown) await writeFile(args.markdown, markdown); +if (args.json) await writeFile(args.json, JSON.stringify(json, null, 2) + '\n'); +if (!args.markdown) process.stdout.write(markdown); + +if (process.env.GITHUB_ACTIONS && changes.length) { + for (const change of changes) { + console.log(`::notice title=Public API change::${verb[change.status]}: ${change.files.join(', ')}`); + } +} diff --git a/bin/bench-browser.mjs b/bin/bench-browser.mjs index c58ce49..2b2f8d8 100644 --- a/bin/bench-browser.mjs +++ b/bin/bench-browser.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Runs the base vs head benchmarks of bin/bench-compare.mjs in a browser and +// Runs the base vs head benchmarks of bin/bench-cases.mjs in a browser and // writes the samples to a JSON file, for `bench-compare.mjs --runtime `. // // Usage: @@ -16,7 +16,7 @@ import { spawn } from 'node:child_process'; import { mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; +import { cpus, tmpdir } from 'node:os'; import { join, relative, resolve, sep } from 'node:path'; import { parseArgs } from 'node:util'; import { ROOT, createStaticServer } from '../test/browser/server.mjs'; @@ -67,10 +67,19 @@ function urlPath(dir) { return `/${path.split(sep).join('/')}`; } +const started = performance.now(); +const elapsed = () => `${((performance.now() - started) / 1000).toFixed(1)} s`; +let ready = false; + const results = Promise.withResolvers(); const server = createStaticServer({ onPost(pathname, message) { if (pathname !== '/results') return; + // the page posts its first message once the browser runs it + if (!ready) { + ready = true; + console.error(`${args.browser}: page ready after ${elapsed()}`); + } if (message.type === 'progress') console.error(message.message); if (message.type === 'done') results.resolve(message.results); if (message.type === 'error') results.reject(new Error(message.message)); @@ -110,7 +119,7 @@ const timer = setTimeout( ); try { - const json = JSON.stringify(await results.promise, null, 2) + '\n'; + const json = JSON.stringify({ ...(await results.promise), cpu: cpus()[0]?.model }, null, 2) + '\n'; if (args.json) { await writeFile(args.json, json); } else { @@ -121,18 +130,25 @@ try { process.exitCode = 1; } finally { clearTimeout(timer); + // SIGKILL: the results are in (or the run failed), so nothing in the + // browser needs a clean shutdown, and Firefox can take seconds on SIGTERM if (browser && browser.exitCode === null && browser.signalCode === null) { + const closing = performance.now(); browser.removeAllListeners('exit'); + let timeout; const exited = new Promise((done) => { browser.once('exit', done); - setTimeout(done, 5000); + // unref: a pending timer would keep the process alive after the exit + timeout = setTimeout(done, 5000).unref(); }); try { - process.kill(-browser.pid, 'SIGTERM'); + process.kill(-browser.pid, 'SIGKILL'); } catch { - browser.kill('SIGTERM'); + browser.kill('SIGKILL'); } await exited; + clearTimeout(timeout); + console.error(`${args.browser}: closed in ${((performance.now() - closing) / 1000).toFixed(1)} s`); } server.closeAllConnections(); server.close(); diff --git a/bin/bench-cases.mjs b/bin/bench-cases.mjs index 60ed7b2..a604666 100644 --- a/bin/bench-cases.mjs +++ b/bin/bench-cases.mjs @@ -1,4 +1,4 @@ -// Benchmark cases shared by bin/bench-compare.mjs (Node) and +// Benchmark cases shared by bin/bench-node.mjs (Node) and // test/browser/compare.worker.mjs (browsers), so every runtime runs the // same calls on the same input. diff --git a/bin/bench-compare.mjs b/bin/bench-compare.mjs old mode 100644 new mode 100755 index 686e334..8cab633 --- a/bin/bench-compare.mjs +++ b/bin/bench-compare.mjs @@ -1,69 +1,25 @@ #!/usr/bin/env node -// Compares the performance of two rregex builds (base vs head), and of head -// vs the native RegExp, and writes a Markdown report and a JSON file with the -// results. +// Writes the report of the benchmarks: compares two rregex builds (base vs +// head) in each runtime, and head vs the native RegExp, as Markdown and JSON. // // Usage: // node bin/bench-compare.mjs --base --head \ -// [--base-label ] [--head-label ] \ -// [--runtime ]... \ -// [--markdown ] [--json ] \ -// [--rounds 5] [--time 250] [--threshold 10] +// --runtime [--runtime ]... \ +// [--base-label ] [--head-label ] [--build-time ] \ +// [--markdown ] [--json ] [--threshold 10] // -// The cases run in Node. Each --runtime adds the results of another runtime, -// written by bin/bench-browser.mjs, to the report. +// Each --runtime is the samples of one runtime, written by bin/bench-node.mjs +// or bin/bench-browser.mjs. The runtimes can run on different machines: every +// delta compares base and head on the same machine. --build-time adds the +// build time of head, written by bin/build.sh (BUILD_TIMINGS=). The lib +// dirs only give the size of the wasm files. // -// To reduce noise, base and head run next to each other for every case, the -// order alternates on each round, and the reported delta is the median of the -// per-round head/base ratios. -// -// Every (round, case) pair runs in a new worker thread. This gives each case a -// clean wasm memory: older builds use the `wee_alloc` allocator, where calls -// with large inputs grow the wasm memory until it hits the 4 GiB limit and trap. +// The reported delta is the median of the per-round head/base ratios. -import { Bench } from 'tinybench'; import { parseArgs } from 'node:util'; import { readFile, writeFile, stat } from 'node:fs/promises'; -import { resolve, dirname } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; -import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads'; -import { - DATA_PATH, - addRound, - cases, - emptySamples, - nativeCases, - roundOrder, - sample, - summarize, -} from './bench-cases.mjs'; - -const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); -const data = sample(await readFile(resolve(root, DATA_PATH), 'utf8')); - -const importLib = (dir) => import(pathToFileURL(resolve(dir, 'esm.mjs')).href); - -if (!isMainThread) { - // Worker: run one case for both builds and the native RegExp, in the given order. - const { dirs, name, order, time } = workerData; - const bench = new Bench({ time, warmupTime: Math.min(100, time) }); - for (const build of order) { - const fn = - build === 'native' - ? nativeCases(data).get(name) - : cases(await importLib(dirs[build]), data).get(name); - if (fn) bench.add(build, fn); - } - await bench.warmup(); - await bench.run(); - const result = {}; - for (const task of bench.tasks) { - if (task.result?.error) throw task.result.error; - result[task.name] = { mean: task.result.mean, rme: task.result.rme }; - } - parentPort.postMessage(result); - process.exit(0); -} +import { resolve } from 'node:path'; +import { summarize } from './bench-cases.mjs'; const { values: args } = parseArgs({ options: { @@ -72,21 +28,18 @@ const { values: args } = parseArgs({ 'base-label': { type: 'string', default: 'base' }, 'head-label': { type: 'string', default: 'head' }, runtime: { type: 'string', multiple: true, default: [] }, + 'build-time': { type: 'string' }, markdown: { type: 'string' }, json: { type: 'string' }, - rounds: { type: 'string', default: process.env.BENCH_ROUNDS ?? '5' }, - time: { type: 'string', default: process.env.BENCH_TIME ?? '250' }, threshold: { type: 'string', default: process.env.BENCH_THRESHOLD ?? '10' }, }, }); -if (!args.base || !args.head) { - console.error('Usage: bench-compare.mjs --base --head [options]'); +if (!args.base || !args.head || !args.runtime.length) { + console.error('Usage: bench-compare.mjs --base --head --runtime ... [options]'); process.exit(2); } -const ROUNDS = Number(args.rounds); -const TIME = Number(args.time); const THRESHOLD = Number(args.threshold) / 100; const dirs = { base: resolve(args.base), head: resolve(args.head) }; @@ -96,36 +49,20 @@ const size = { }; size.delta = size.head / size.base - 1; -const names = [...cases(await importLib(dirs.head), data).keys()]; - -/** Runs one case for both builds in a new worker thread. */ -const runCase = (name, order) => - new Promise((done, fail) => { - const worker = new Worker(fileURLToPath(import.meta.url), { - workerData: { dirs, name, order, time: TIME }, - }); - worker.once('message', done); - worker.once('error', fail); - worker.once('exit', (code) => code && fail(new Error(`${name}: worker exited with code ${code}`))); - }); - -const samples = Object.fromEntries(names.map((name) => [name, emptySamples()])); - -for (let round = 0; round < ROUNDS; round++) { - for (const name of names) { - addRound(samples[name], await runCase(name, roundOrder(round))); - } - console.error(`round ${round + 1}/${ROUNDS} done`); -} +const build = args['build-time'] ? JSON.parse(await readFile(args['build-time'], 'utf8')) : null; -const runtimes = [ - { runtime: `Node ${process.version}`, rounds: ROUNDS, time: TIME, results: summarize(samples, THRESHOLD) }, -]; +const runtimes = []; for (const file of args.runtime) { - const other = JSON.parse(await readFile(file, 'utf8')); - runtimes.push({ ...other, samples: undefined, results: summarize(other.samples, THRESHOLD) }); + const { samples, ...runtime } = JSON.parse(await readFile(file, 'utf8')); + runtimes.push({ ...runtime, results: summarize(samples, THRESHOLD) }); } +// every case of every runtime, in the order of the first one that has it +const names = [...new Set(runtimes.flatMap(({ results }) => results.map((r) => r.name)))]; + +// "5 rounds × 250 ms", once if every runtime used the same settings +const settings = [...new Set(runtimes.map((r) => `${r.rounds} rounds × ${r.time} ms`))]; + const regressions = runtimes.flatMap(({ runtime, results }) => results.filter((r) => r.status === 'regression').map((r) => ({ runtime, ...r })), ); @@ -150,7 +87,7 @@ const lines = []; lines.push('## ⏱️ Benchmark results', ''); lines.push( `Comparing **head** ${args['head-label']} against **base** ${args['base-label']} ` + - `on ${list(runtimes.map((r) => r.runtime))} (${ROUNDS} rounds × ${TIME} ms per case). ` + + `on ${list(runtimes.map((r) => r.runtime))} (${settings.join(' / ')} per case). ` + `Changes within ±${args.threshold}% are treated as noise.`, '', ); @@ -186,8 +123,8 @@ for (const name of names) { lines.push(''); // details: the times of each runtime -for (const { runtime, results } of runtimes) { - lines.push(`
${runtime}: times`, ''); +for (const { runtime, cpu, results } of runtimes) { + lines.push(`
${runtime}: times${cpu ? ` (${cpu})` : ''}`, ''); lines.push('| | Benchmark | base | head | RegExp | Δ time | ± |'); lines.push('|:-:|:--|--:|--:|--:|--:|--:|'); for (const r of results) { @@ -206,16 +143,29 @@ lines.push('| Artifact | base | head | Δ size |'); lines.push('|:--|--:|--:|--:|'); lines.push(`| \`rregex.wasm\` | ${fmtBytes(size.base)} | ${fmtBytes(size.head)} | ${fmtDelta(size.delta)} |`); lines.push(''); + +// build time of head (base is the published package, it is not built) +if (build) { + lines.push('| Build step | head |', '|:--|--:|'); + const seconds = (ms) => `${(ms / 1000).toFixed(2)} s`; + for (const { name, ms } of build.steps) lines.push(`| \`${name}\` | ${seconds(ms)} |`); + lines.push(`| **Total** | **${seconds(build.total)}** |`, ''); +} + lines.push( 'Δ time is the median of the per-round head/base ratios (lower is better). ' + 'RegExp is the time of the equivalent native `RegExp` call in the same runtime ' + '(`new` has no equivalent, `RegExp` compiles on first use). ' + '± is the worst relative margin of error seen across rounds. ' + + (build + ? 'The build time of `cargo build` depends on the state of the CI cache; ' + + '`wasm-opt` and `tsdown` run in parallel, so the total is less than the sum. ' + : '') + 'Shared CI runners are noisy: re-run the job before you act on a small regression.', ); const markdown = lines.join('\n') + '\n'; -const json = { rounds: ROUNDS, time: TIME, threshold: THRESHOLD, size, runtimes }; +const json = { threshold: THRESHOLD, size, build, runtimes }; if (args.markdown) await writeFile(args.markdown, markdown); if (args.json) await writeFile(args.json, JSON.stringify(json, null, 2) + '\n'); diff --git a/bin/bench-node.mjs b/bin/bench-node.mjs new file mode 100755 index 0000000..b62ea57 --- /dev/null +++ b/bin/bench-node.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node +// Runs the base vs head benchmarks in Node and writes the samples to a JSON +// file, for `bench-compare.mjs --runtime ` (same format as +// bin/bench-browser.mjs). +// +// Usage: +// node bin/bench-node.mjs --base --head \ +// [--json ] [--rounds 5] [--time 250] +// +// To reduce noise, base and head run next to each other for every case and +// the order alternates on each round (see bin/bench-cases.mjs). +// +// Every (round, case) pair runs in a new worker thread. This gives each case a +// clean wasm memory: older builds use the `wee_alloc` allocator, where calls +// with large inputs grow the wasm memory until it hits the 4 GiB limit and trap. + +import { Bench } from 'tinybench'; +import { parseArgs } from 'node:util'; +import { readFile, writeFile } from 'node:fs/promises'; +import { cpus } from 'node:os'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads'; +import { DATA_PATH, addRound, cases, emptySamples, nativeCases, roundOrder, sample } from './bench-cases.mjs'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const data = sample(await readFile(resolve(root, DATA_PATH), 'utf8')); + +const importLib = (dir) => import(pathToFileURL(resolve(dir, 'esm.mjs')).href); + +if (!isMainThread) { + // Worker: run one case for both builds and the native RegExp, in the given order. + const { dirs, name, order, time } = workerData; + const bench = new Bench({ time, warmupTime: Math.min(100, time) }); + for (const build of order) { + const fn = + build === 'native' + ? nativeCases(data).get(name) + : cases(await importLib(dirs[build]), data).get(name); + if (fn) bench.add(build, fn); + } + await bench.warmup(); + await bench.run(); + const result = {}; + for (const task of bench.tasks) { + if (task.result?.error) throw task.result.error; + result[task.name] = { mean: task.result.mean, rme: task.result.rme }; + } + parentPort.postMessage(result); + process.exit(0); +} + +const { values: args } = parseArgs({ + options: { + base: { type: 'string' }, + head: { type: 'string' }, + json: { type: 'string' }, + rounds: { type: 'string', default: process.env.BENCH_ROUNDS ?? '5' }, + time: { type: 'string', default: process.env.BENCH_TIME ?? '250' }, + }, +}); + +if (!args.base || !args.head) { + console.error('Usage: bench-node.mjs --base --head [--json ] [options]'); + process.exit(2); +} + +const rounds = Number(args.rounds); +const time = Number(args.time); +const runtime = `Node ${process.version}`; +const dirs = { base: resolve(args.base), head: resolve(args.head) }; +const names = [...cases(await importLib(dirs.head), data).keys()]; + +/** Runs one case for both builds in a new worker thread. */ +const runCase = (name, order) => + new Promise((done, fail) => { + const worker = new Worker(fileURLToPath(import.meta.url), { + workerData: { dirs, name, order, time }, + }); + worker.once('message', done); + worker.once('error', fail); + worker.once('exit', (code) => code && fail(new Error(`${name}: worker exited with code ${code}`))); + }); + +console.error(`${runtime}: ${names.length} cases × ${rounds} rounds × ${time} ms`); + +const samples = Object.fromEntries(names.map((name) => [name, emptySamples()])); +for (let round = 0; round < rounds; round++) { + for (const name of names) { + addRound(samples[name], await runCase(name, roundOrder(round))); + } + console.error(`${runtime}: round ${round + 1}/${rounds} done`); +} + +const results = { runtime, cpu: cpus()[0]?.model, rounds, time, samples }; +const json = JSON.stringify(results, null, 2) + '\n'; +if (args.json) { + await writeFile(args.json, json); +} else { + process.stdout.write(json); +} diff --git a/bin/build.sh b/bin/build.sh index 5fb4ed7..3ca2c70 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -1,4 +1,7 @@ #!/bin/bash +# Builds lib/. Prints the time of each step. With BUILD_TIMINGS=, it also +# writes them to as JSON: { "steps": [{ "name", "ms" }], "total": ms }. +# wasm-opt and tsdown run in parallel, so the total is less than the sum. set -euo pipefail cd "$(dirname "$0")/.." @@ -6,22 +9,77 @@ cd "$(dirname "$0")/.." TARGET=wasm32-unknown-unknown WASM="target/$TARGET/release/rregex.wasm" +# milliseconds; whole seconds where `date` has no %N +now_ms() { + local ns + ns=$(date +%s%N) + case "$ns" in + *[!0-9]*) echo $(($(date +%s) * 1000)) ;; + *) echo $((ns / 1000000)) ;; + esac +} + +seconds() { + echo "$(($1 / 1000)).$(printf '%03d' $(($1 % 1000)))s" +} + +mkdir -p target +TIMES=target/build-times.txt +: > "$TIMES" + +# timed : runs the command and records its time +timed() { + local name=$1 start ms + shift + start=$(now_ms) + "$@" + ms=$(($(now_ms) - start)) + echo "$name $ms" >> "$TIMES" + echo " ⏱️ $name: $(seconds "$ms")" +} + +bindings() { + rm -rf js/wasm lib + wasm-bindgen --target web --omit-default-module-path \ + --out-dir js/wasm --out-name rregex "$WASM" + node bin/metadata.mjs + mkdir -p lib +} + +# tsdown.config.ts does not generate these declarations: same as core.d.mts +bundle() { + pnpm exec tsdown --silent + for dts in core.d.cts commonjs.d.cts cf.d.mts; do + cp lib/core.d.mts "lib/$dts" + done +} + +STARTED=$(now_ms) + echo " 🦀 Compiling wasm..." -cargo build --release --target "$TARGET" +timed "cargo build" cargo build --release --target "$TARGET" echo " 🔗 Generating bindings..." -rm -rf js/wasm lib -wasm-bindgen --target web --omit-default-module-path \ - --out-dir js/wasm --out-name rregex "$WASM" -node bin/metadata.mjs +timed "wasm-bindgen" bindings + +# tsdown only reads js/: lib/rregex.wasm is never bundled +echo " 🗜️ Optimizing wasm and 📦 bundling..." +timed "wasm-opt" ./bin/wasm-opt.sh js/wasm/rregex_bg.wasm lib/rregex.wasm & +WASM_OPT=$! +timed "tsdown" bundle +wait "$WASM_OPT" -echo " 🗜️ Optimizing wasm..." -mkdir -p lib -pnpm exec wasm-opt -O --enable-reference-types --enable-bulk-memory \ - --enable-nontrapping-float-to-int --enable-sign-ext --enable-mutable-globals \ - js/wasm/rregex_bg.wasm -o lib/rregex.wasm +TOTAL=$(($(now_ms) - STARTED)) -echo " 📦 Bundling..." -pnpm exec tsdown --silent +if [ -n "${BUILD_TIMINGS:-}" ]; then + STEPS=() + for name in "cargo build" "wasm-bindgen" "wasm-opt" "tsdown"; do + STEPS+=("{\"name\":\"$name\",\"ms\":$(grep "^$name " "$TIMES" | awk '{print $NF}')}") + done + ( + IFS=, + echo "{\"steps\":[${STEPS[*]}],\"total\":$TOTAL}" + ) > "$BUILD_TIMINGS" +fi -echo " ✅ Done!" +echo " ✅ Done in $(seconds "$TOTAL")!" diff --git a/bin/install-binaryen.sh b/bin/install-binaryen.sh new file mode 100755 index 0000000..3009a84 --- /dev/null +++ b/bin/install-binaryen.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Usage: bin/install-binaryen.sh +# +# Downloads the native binaryen release with the same version as the +# `binaryen` npm package to , checks its sha256 and prints its bin +# directory. bin/wasm-opt.sh uses that `wasm-opt` when it is in PATH. +set -euo pipefail + +cd "$(dirname "$0")/.." + +DIR=$1 +VERSION="version_$(node -p 'require("./node_modules/binaryen/package.json").version.split(".")[0]')" + +case "$(uname -s)-$(uname -m)" in + Linux-x86_64) PLATFORM=x86_64-linux ;; + Linux-aarch64 | Linux-arm64) PLATFORM=aarch64-linux ;; + Darwin-x86_64) PLATFORM=x86_64-macos ;; + Darwin-arm64) PLATFORM=arm64-macos ;; + *) + echo "no binaryen release for $(uname -s) $(uname -m)" >&2 + exit 1 + ;; +esac + +TARBALL="binaryen-$VERSION-$PLATFORM.tar.gz" +URL="https://github.com/WebAssembly/binaryen/releases/download/$VERSION/$TARBALL" + +mkdir -p "$DIR" +cd "$DIR" +curl -fsSLO "$URL" +curl -fsSLO "$URL.sha256" +if command -v sha256sum > /dev/null; then + sha256sum -c "$TARBALL.sha256" >&2 +else + shasum -a 256 -c "$TARBALL.sha256" >&2 +fi +tar xzf "$TARBALL" +rm "$TARBALL" "$TARBALL.sha256" + +echo "$PWD/binaryen-$VERSION/bin" diff --git a/bin/install.sh b/bin/install.sh index 5d4b15a..5e655d2 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -17,13 +17,23 @@ echo "" rustup target add wasm32-unknown-unknown +## prebuilt binaries with cargo-binstall when it is installed; +## `cargo install` compiles each one (~1-2 min) +cargo_install() { + if cargo binstall -V > /dev/null 2>&1; then + cargo binstall --no-confirm "$@" + else + cargo install "$@" + fi +} + ## wasm-bindgen-cli must match the wasm-bindgen crate version in Cargo.lock WASM_BINDGEN_REQUIRED=$(grep -A1 '^name = "wasm-bindgen"$' Cargo.lock | sed -n 's/^version = "\(.*\)"$/\1/p') WASM_BINDGEN_VERSION=$(wasm-bindgen --version 2> /dev/null | cut -d' ' -f2 || true) if [ "$WASM_BINDGEN_VERSION" != "$WASM_BINDGEN_REQUIRED" ]; then echo " 📥 installing wasm-bindgen-cli $WASM_BINDGEN_REQUIRED" - cargo install wasm-bindgen-cli --version "$WASM_BINDGEN_REQUIRED" --locked --force + cargo_install wasm-bindgen-cli --version "$WASM_BINDGEN_REQUIRED" --locked --force fi echo " ✅ wasm-bindgen $(wasm-bindgen --version | cut -d' ' -f2) ($(which wasm-bindgen))" @@ -34,9 +44,21 @@ WASM_PACK_VERSION=$(wasm-pack --version 2> /dev/null || true) ## wasm-pack is only used to run the Rust tests (pnpm test:rs) if [ "$WASM_PACK_VERSION" = "" ]; then echo " 📥 installing wasm-pack" - cargo install wasm-pack + cargo_install wasm-pack WASM_PACK_VERSION=$(wasm-pack --version) fi echo " ✅ $WASM_PACK_VERSION ($(which wasm-pack))" echo "" + +## native wasm-opt with the version of the binaryen npm package (bin/wasm-opt.sh) +BINARYEN_REQUIRED="version_$(node -p 'require("./node_modules/binaryen/package.json").version.split(".")[0]')" +WASM_OPT="target/binaryen/binaryen-$BINARYEN_REQUIRED/bin/wasm-opt" + +if [ ! -x "$WASM_OPT" ]; then + echo " 📥 installing binaryen $BINARYEN_REQUIRED" + ./bin/install-binaryen.sh target/binaryen > /dev/null +fi + +echo " ✅ $("$WASM_OPT" --version) ($WASM_OPT)" +echo "" diff --git a/bin/wasm-opt.sh b/bin/wasm-opt.sh new file mode 100755 index 0000000..bac8d28 --- /dev/null +++ b/bin/wasm-opt.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Usage: bin/wasm-opt.sh +# +# Optimizes the wasm with binaryen. It prefers a native `wasm-opt` with the +# same version as the `binaryen` npm package: same output, but multi-threaded +# and ~10x faster than the npm package, which is an emscripten (JS) build. +# It looks for it in PATH and in target/binaryen (`pnpm setup` installs it +# there). Without one, it uses the npm package. +# +# The last output is kept in target/wasm-opt and reused while the input does +# not change. +# +# WASM_OPT_NPM=1 always runs the npm package, without the cache. +set -euo pipefail + +SELF="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")" +cd "$(dirname "$0")/.." + +FLAGS=( + -O + --enable-reference-types + --enable-bulk-memory + --enable-nontrapping-float-to-int + --enable-sign-ext + --enable-mutable-globals +) + +VERSION=$(node -p 'require("./node_modules/binaryen/package.json").version.split(".")[0]') + +# `pnpm run` and `node --run` put node_modules/.bin first in PATH, so skip it +native_wasm_opt() { + local bin + while read -r bin; do + case "$bin" in */node_modules/*) continue ;; esac + if [[ "$("$bin" --version 2> /dev/null)" == *"(version_$VERSION)"* ]]; then + echo "$bin" + return + fi + done < <( + type -ap wasm-opt || true + echo "$PWD/target/binaryen/binaryen-version_$VERSION/bin/wasm-opt" + ) +} + +sha256() { + if command -v sha256sum > /dev/null; then sha256sum; else shasum -a 256; fi | cut -d' ' -f1 +} + +# The output only depends on the input, the binaryen version and the flags in +# this file: reuse the last output when none of them changed +CACHE_DIR=target/wasm-opt +CACHE="$CACHE_DIR/$({ sha256 < "$1"; echo "$VERSION"; sha256 < "$SELF"; } | sha256).wasm" + +if [ "${WASM_OPT_NPM:-}" != "1" ] && [ -f "$CACHE" ]; then + echo " input did not change, reusing $CACHE" + cp "$CACHE" "$2" + exit 0 +fi + +NATIVE="" +if [ "${WASM_OPT_NPM:-}" != "1" ]; then + NATIVE=$(native_wasm_opt) +fi + +if [ -n "$NATIVE" ]; then + echo " native wasm-opt version_$VERSION ($NATIVE)" + "$NATIVE" "${FLAGS[@]}" "$1" -o "$2" +else + if [ "${WASM_OPT_NPM:-}" != "1" ]; then + echo " native wasm-opt version_$VERSION not found, using the npm package (slower, run \`pnpm setup\`)" + fi + pnpm exec wasm-opt "${FLAGS[@]}" "$1" -o "$2" +fi + +if [ "${WASM_OPT_NPM:-}" != "1" ]; then + rm -rf "$CACHE_DIR" + mkdir -p "$CACHE_DIR" + cp "$2" "$CACHE" +fi diff --git a/test/browser/bench.html b/test/browser/bench.html index 729931c..0e8e385 100644 --- a/test/browser/bench.html +++ b/test/browser/bench.html @@ -66,7 +66,7 @@

Same patterns and input (test/__data__/input-text.txt) as - bin/bench-compare.mjs. Both engines compile the pattern once, before the + bin/bench-cases.mjs. Both engines compile the pattern once, before the benchmark. \w is ASCII in RegExp and Unicode in RRegex. Hover a bar for its numbers.

diff --git a/test/browser/compare.mjs b/test/browser/compare.mjs index e349183..47a187e 100644 --- a/test/browser/compare.mjs +++ b/test/browser/compare.mjs @@ -1,4 +1,4 @@ -// Browser version of bin/bench-compare.mjs: runs every case for base, head +// Browser version of bin/bench-node.mjs: runs every case for base, head // and the native RegExp, and reports the samples to `?report=` (used by // bin/bench-browser.mjs) or as `globalThis.compareResults`. // diff --git a/test/browser/compare.worker.mjs b/test/browser/compare.worker.mjs index c2e6242..4b5dd66 100644 --- a/test/browser/compare.worker.mjs +++ b/test/browser/compare.worker.mjs @@ -1,6 +1,6 @@ // Runs one case for both builds and the native RegExp, in the given order. // Each case runs in a new worker, so every build starts with a clean wasm -// memory (see bin/bench-compare.mjs). +// memory (see bin/bench-node.mjs). import { cases, nativeCases } from "../../bin/bench-cases.mjs" import { measure } from "./measure.mjs" diff --git a/tsdown.config.ts b/tsdown.config.ts index 9a062bc..9e325a0 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -2,6 +2,10 @@ import { defineConfig, type UserConfig } from "tsdown"; // Every entry is one self-contained file in lib/ next to lib/rregex.wasm. // bin/build.sh generates js/wasm/ (wasm-bindgen glue and metadata) first. +// +// commonjs, core (cjs) and cf export exactly what core exports, so their +// declarations are the same as core.d.mts: bin/build.sh copies it instead of +// generating them (`dts: false`). const shared = { outDir: "lib", clean: false, @@ -29,6 +33,7 @@ export default defineConfig([ entry: { commonjs: "js/node.ts" }, platform: "node", format: "cjs", + dts: false, outExtensions: ext(".cjs", ".d.cts"), }, // Any runtime, the user loads the module @@ -44,6 +49,7 @@ export default defineConfig([ entry: { core: "js/core.ts" }, platform: "neutral", format: "cjs", + dts: false, outExtensions: ext(".cjs", ".d.cts"), }, // Browsers and bundlers @@ -60,6 +66,7 @@ export default defineConfig([ entry: { cf: "js/workerd.ts" }, platform: "neutral", format: "esm", + dts: false, deps: { onlyBundle: false, neverBundle: [/\.wasm$/] }, outExtensions: ext(".mjs", ".d.mts"), },