From 53b556998e67b57529a7764ce4a6ef2de5baf8dd Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 01:07:51 -0300 Subject: [PATCH 01/12] build: use a native wasm-opt and report the build time in the benchmark The `binaryen` npm package is an emscripten (JS) build of wasm-opt and runs on one thread: ~8.3s of a ~9s rebuild. bin/wasm-opt.sh now prefers a native wasm-opt with the same version as the npm package from PATH (~0.6s, same output byte for byte) and falls back to the npm package. - bin/install-binaryen.sh downloads the matching binaryen release and checks its sha256; both workflows put it in PATH - the release workflow checks that the native and the npm wasm-opt give the same wasm - bin/build.sh prints the time of each step and writes them to $BUILD_TIMINGS; the benchmark report shows them Co-Authored-By: Claude Opus 5.5 --- .github/workflows/benchmark.yml | 12 +++++-- .github/workflows/release.yml | 12 +++++++ bin/bench-compare.mjs | 20 ++++++++++-- bin/build.sh | 56 ++++++++++++++++++++++++++++----- bin/install-binaryen.sh | 40 +++++++++++++++++++++++ bin/wasm-opt.sh | 50 +++++++++++++++++++++++++++++ 6 files changed, 177 insertions(+), 13 deletions(-) create mode 100755 bin/install-binaryen.sh create mode 100755 bin/wasm-opt.sh diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 3e9db00..8f2944b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -16,6 +16,8 @@ on: - "package.json" - "pnpm-lock.yaml" - "bin/build.sh" + - "bin/wasm-opt.sh" + - "bin/install-binaryen.sh" - "js/**" - "tsdown.config.ts" - "bin/bench-*.mjs" @@ -89,10 +91,15 @@ jobs: pnpm install --frozen-lockfile node --run setup - # check that the lib is complete + # native wasm-opt, see the release workflow + - name: installing binaryen + run: ./bin/install-binaryen.sh "$RUNNER_TEMP/binaryen" >> "$GITHUB_PATH" + + # check that the lib is complete; the build time goes to the report - name: building head run: | - node --run build + mkdir -p "$RUNNER_TEMP/benchmark" + BUILD_TIMINGS="$RUNNER_TEMP/benchmark/build.json" node --run build test -f lib/esm.mjs -a -f lib/rregex.wasm # Chrome and Firefox come with the runner image; the runtimes run one @@ -116,6 +123,7 @@ jobs: --head-label "\`${HEAD_SHA::7}\`" \ --runtime "$REPORT_DIR/chrome.json" \ --runtime "$REPORT_DIR/firefox.json" \ + --build-time "$REPORT_DIR/build.json" \ --markdown "$REPORT_DIR/report.md" \ --json "$REPORT_DIR/results.json" cat "$REPORT_DIR/report.md" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 06fa6d7..0796bc1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,6 +83,11 @@ jobs: - name: installing run: pnpm install --frozen-lockfile + # native wasm-opt with the version of the binaryen npm package, used by + # bin/wasm-opt.sh: same output, ~10x faster than the npm (JS) build + - name: installing binaryen + run: ./bin/install-binaryen.sh "$RUNNER_TEMP/binaryen" >> "$GITHUB_PATH" + - name: setup run: node --run setup @@ -97,6 +102,13 @@ jobs: name: lib path: lib/ + # the native wasm-opt must give the same wasm as the npm package + - name: checking wasm-opt + run: | + wasm-opt --version + WASM_OPT_NPM=1 ./bin/wasm-opt.sh js/wasm/rregex_bg.wasm target/rregex.npm.wasm + cmp target/rregex.npm.wasm lib/rregex.wasm + test_node: runs-on: ubuntu-24.04 diff --git a/bin/bench-compare.mjs b/bin/bench-compare.mjs index 686e334..334ed27 100644 --- a/bin/bench-compare.mjs +++ b/bin/bench-compare.mjs @@ -6,12 +6,13 @@ // Usage: // node bin/bench-compare.mjs --base --head \ // [--base-label ] [--head-label ] \ -// [--runtime ]... \ +// [--runtime ]... [--build-time ] \ // [--markdown ] [--json ] \ // [--rounds 5] [--time 250] [--threshold 10] // // The cases run in Node. Each --runtime adds the results of another runtime, -// written by bin/bench-browser.mjs, to the report. +// written by bin/bench-browser.mjs, to the report. --build-time adds the build +// time of head, written by bin/build.sh (BUILD_TIMINGS=). // // To reduce noise, base and head run next to each other for every case, the // order alternates on each round, and the reported delta is the median of the @@ -72,6 +73,7 @@ const { values: args } = parseArgs({ 'base-label': { type: 'string', default: 'base' }, 'head-label': { type: 'string', default: 'head' }, runtime: { type: 'string', multiple: true, default: [] }, + 'build-time': { type: 'string' }, markdown: { type: 'string' }, json: { type: 'string' }, rounds: { type: 'string', default: process.env.BENCH_ROUNDS ?? '5' }, @@ -96,6 +98,8 @@ const size = { }; size.delta = size.head / size.base - 1; +const build = args['build-time'] ? JSON.parse(await readFile(args['build-time'], 'utf8')) : null; + const names = [...cases(await importLib(dirs.head), data).keys()]; /** Runs one case for both builds in a new worker thread. */ @@ -206,16 +210,26 @@ lines.push('| Artifact | base | head | ฮ” size |'); lines.push('|:--|--:|--:|--:|'); lines.push(`| \`rregex.wasm\` | ${fmtBytes(size.base)} | ${fmtBytes(size.head)} | ${fmtDelta(size.delta)} |`); lines.push(''); + +// build time of head (base is the published package, it is not built) +if (build) { + lines.push('| Build step | head |', '|:--|--:|'); + const seconds = (ms) => `${(ms / 1000).toFixed(2)} s`; + for (const { name, ms } of build.steps) lines.push(`| \`${name}\` | ${seconds(ms)} |`); + lines.push(`| **Total** | **${seconds(build.total)}** |`, ''); +} + lines.push( 'ฮ” time is the median of the per-round head/base ratios (lower is better). ' + 'RegExp is the time of the equivalent native `RegExp` call in the same runtime ' + '(`new` has no equivalent, `RegExp` compiles on first use). ' + 'ยฑ is the worst relative margin of error seen across rounds. ' + + (build ? 'The build time of `cargo build` depends on the state of the CI cache. ' : '') + 'Shared CI runners are noisy: re-run the job before you act on a small regression.', ); const markdown = lines.join('\n') + '\n'; -const json = { rounds: ROUNDS, time: TIME, threshold: THRESHOLD, size, runtimes }; +const json = { rounds: ROUNDS, time: TIME, threshold: THRESHOLD, size, build, runtimes }; if (args.markdown) await writeFile(args.markdown, markdown); if (args.json) await writeFile(args.json, JSON.stringify(json, null, 2) + '\n'); diff --git a/bin/build.sh b/bin/build.sh index 5fb4ed7..659c7c4 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -1,4 +1,6 @@ #!/bin/bash +# Builds lib/. Prints the time of each step. With BUILD_TIMINGS=, it also +# writes them to as JSON: { "steps": [{ "name", "ms" }], "total": ms }. set -euo pipefail cd "$(dirname "$0")/.." @@ -6,22 +8,60 @@ cd "$(dirname "$0")/.." TARGET=wasm32-unknown-unknown WASM="target/$TARGET/release/rregex.wasm" -echo " ๐Ÿฆ€ Compiling wasm..." +# milliseconds; whole seconds where `date` has no %N +now_ms() { + local ns + ns=$(date +%s%N) + case "$ns" in + *[!0-9]*) echo $(($(date +%s) * 1000)) ;; + *) echo $((ns / 1000000)) ;; + esac +} + +STEPS=() +STARTED=$(now_ms) +STEP_NAME="" +STEP_START=0 + +step() { + end_step + echo " $1" + STEP_NAME=$2 + STEP_START=$(now_ms) +} + +end_step() { + [ -n "$STEP_NAME" ] || return 0 + local ms=$(($(now_ms) - STEP_START)) + STEPS+=("{\"name\":\"$STEP_NAME\",\"ms\":$ms}") + echo " โฑ๏ธ $STEP_NAME: $((ms / 1000)).$(printf '%03d' $((ms % 1000)))s" + STEP_NAME="" +} + +step "๐Ÿฆ€ Compiling wasm..." "cargo build" cargo build --release --target "$TARGET" -echo " ๐Ÿ”— Generating bindings..." +step "๐Ÿ”— Generating bindings..." "wasm-bindgen" rm -rf js/wasm lib wasm-bindgen --target web --omit-default-module-path \ --out-dir js/wasm --out-name rregex "$WASM" node bin/metadata.mjs -echo " ๐Ÿ—œ๏ธ Optimizing wasm..." +step "๐Ÿ—œ๏ธ Optimizing wasm..." "wasm-opt" mkdir -p lib -pnpm exec wasm-opt -O --enable-reference-types --enable-bulk-memory \ - --enable-nontrapping-float-to-int --enable-sign-ext --enable-mutable-globals \ - js/wasm/rregex_bg.wasm -o lib/rregex.wasm +./bin/wasm-opt.sh js/wasm/rregex_bg.wasm lib/rregex.wasm -echo " ๐Ÿ“ฆ Bundling..." +step "๐Ÿ“ฆ Bundling..." "tsdown" pnpm exec tsdown --silent -echo " โœ… Done!" +end_step +TOTAL=$(($(now_ms) - STARTED)) + +if [ -n "${BUILD_TIMINGS:-}" ]; then + ( + IFS=, + echo "{\"steps\":[${STEPS[*]}],\"total\":$TOTAL}" + ) > "$BUILD_TIMINGS" +fi + +echo " โœ… Done in $((TOTAL / 1000)).$(printf '%03d' $((TOTAL % 1000)))s!" diff --git a/bin/install-binaryen.sh b/bin/install-binaryen.sh new file mode 100755 index 0000000..3009a84 --- /dev/null +++ b/bin/install-binaryen.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Usage: bin/install-binaryen.sh +# +# Downloads the native binaryen release with the same version as the +# `binaryen` npm package to , checks its sha256 and prints its bin +# directory. bin/wasm-opt.sh uses that `wasm-opt` when it is in PATH. +set -euo pipefail + +cd "$(dirname "$0")/.." + +DIR=$1 +VERSION="version_$(node -p 'require("./node_modules/binaryen/package.json").version.split(".")[0]')" + +case "$(uname -s)-$(uname -m)" in + Linux-x86_64) PLATFORM=x86_64-linux ;; + Linux-aarch64 | Linux-arm64) PLATFORM=aarch64-linux ;; + Darwin-x86_64) PLATFORM=x86_64-macos ;; + Darwin-arm64) PLATFORM=arm64-macos ;; + *) + echo "no binaryen release for $(uname -s) $(uname -m)" >&2 + exit 1 + ;; +esac + +TARBALL="binaryen-$VERSION-$PLATFORM.tar.gz" +URL="https://github.com/WebAssembly/binaryen/releases/download/$VERSION/$TARBALL" + +mkdir -p "$DIR" +cd "$DIR" +curl -fsSLO "$URL" +curl -fsSLO "$URL.sha256" +if command -v sha256sum > /dev/null; then + sha256sum -c "$TARBALL.sha256" >&2 +else + shasum -a 256 -c "$TARBALL.sha256" >&2 +fi +tar xzf "$TARBALL" +rm "$TARBALL" "$TARBALL.sha256" + +echo "$PWD/binaryen-$VERSION/bin" diff --git a/bin/wasm-opt.sh b/bin/wasm-opt.sh new file mode 100755 index 0000000..1771c0e --- /dev/null +++ b/bin/wasm-opt.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Usage: bin/wasm-opt.sh +# +# Optimizes the wasm with binaryen. It prefers a native `wasm-opt` with the +# same version as the `binaryen` npm package: same output, but multi-threaded +# and ~10x faster than the npm package, which is an emscripten (JS) build. +# Without one, it uses the npm package. +# +# WASM_OPT_NPM=1 always uses the npm package. +set -euo pipefail + +cd "$(dirname "$0")/.." + +FLAGS=( + -O + --enable-reference-types + --enable-bulk-memory + --enable-nontrapping-float-to-int + --enable-sign-ext + --enable-mutable-globals +) + +VERSION=$(node -p 'require("./node_modules/binaryen/package.json").version.split(".")[0]') + +# `pnpm run` and `node --run` put node_modules/.bin first in PATH, so skip it +native_wasm_opt() { + local bin + while read -r bin; do + case "$bin" in */node_modules/*) continue ;; esac + if [[ "$("$bin" --version 2> /dev/null)" == *"(version_$VERSION)"* ]]; then + echo "$bin" + return + fi + done < <(type -ap wasm-opt) +} + +NATIVE="" +if [ "${WASM_OPT_NPM:-}" != "1" ]; then + NATIVE=$(native_wasm_opt) +fi + +if [ -n "$NATIVE" ]; then + echo " native wasm-opt version_$VERSION ($NATIVE)" + "$NATIVE" "${FLAGS[@]}" "$1" -o "$2" +else + if [ "${WASM_OPT_NPM:-}" != "1" ]; then + echo " native wasm-opt version_$VERSION not found, using the npm package (slower)" + fi + pnpm exec wasm-opt "${FLAGS[@]}" "$1" -o "$2" +fi From 54443f8ba443ea5d6354499395e12e12e440e5bf Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 01:36:03 -0300 Subject: [PATCH 02/12] ci: remove the wasm-opt output check The native and the npm wasm-opt gave the same wasm on Linux x86_64 in CI. bin/wasm-opt.sh only uses a native wasm-opt with the same version. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0796bc1..e0aea93 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,13 +102,6 @@ jobs: name: lib path: lib/ - # the native wasm-opt must give the same wasm as the npm package - - name: checking wasm-opt - run: | - wasm-opt --version - WASM_OPT_NPM=1 ./bin/wasm-opt.sh js/wasm/rregex_bg.wasm target/rregex.npm.wasm - cmp target/rregex.npm.wasm lib/rregex.wasm - test_node: runs-on: ubuntu-24.04 From 7e0de1bd8ffe03d24720348e4ba53ea7e07cdc44 Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 01:38:40 -0300 Subject: [PATCH 03/12] build: install the native binaryen with `pnpm setup` bin/install.sh installs it in target/binaryen and bin/wasm-opt.sh looks for it there after PATH, so local builds also use the native wasm-opt. The workflows already run `setup`, so their own install step goes away. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/benchmark.yml | 4 ---- .github/workflows/release.yml | 5 ----- bin/install.sh | 12 ++++++++++++ bin/wasm-opt.sh | 10 +++++++--- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8f2944b..8686191 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -91,10 +91,6 @@ jobs: pnpm install --frozen-lockfile node --run setup - # native wasm-opt, see the release workflow - - name: installing binaryen - run: ./bin/install-binaryen.sh "$RUNNER_TEMP/binaryen" >> "$GITHUB_PATH" - # check that the lib is complete; the build time goes to the report - name: building head run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e0aea93..06fa6d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,11 +83,6 @@ jobs: - name: installing run: pnpm install --frozen-lockfile - # native wasm-opt with the version of the binaryen npm package, used by - # bin/wasm-opt.sh: same output, ~10x faster than the npm (JS) build - - name: installing binaryen - run: ./bin/install-binaryen.sh "$RUNNER_TEMP/binaryen" >> "$GITHUB_PATH" - - name: setup run: node --run setup diff --git a/bin/install.sh b/bin/install.sh index 5d4b15a..0417c06 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -40,3 +40,15 @@ fi echo " โœ… $WASM_PACK_VERSION ($(which wasm-pack))" echo "" + +## native wasm-opt with the version of the binaryen npm package (bin/wasm-opt.sh) +BINARYEN_REQUIRED="version_$(node -p 'require("./node_modules/binaryen/package.json").version.split(".")[0]')" +WASM_OPT="target/binaryen/binaryen-$BINARYEN_REQUIRED/bin/wasm-opt" + +if [ ! -x "$WASM_OPT" ]; then + echo " ๐Ÿ“ฅ installing binaryen $BINARYEN_REQUIRED" + ./bin/install-binaryen.sh target/binaryen > /dev/null +fi + +echo " โœ… $("$WASM_OPT" --version) ($WASM_OPT)" +echo "" diff --git a/bin/wasm-opt.sh b/bin/wasm-opt.sh index 1771c0e..ec494cf 100755 --- a/bin/wasm-opt.sh +++ b/bin/wasm-opt.sh @@ -4,7 +4,8 @@ # Optimizes the wasm with binaryen. It prefers a native `wasm-opt` with the # same version as the `binaryen` npm package: same output, but multi-threaded # and ~10x faster than the npm package, which is an emscripten (JS) build. -# Without one, it uses the npm package. +# It looks for it in PATH and in target/binaryen (`pnpm setup` installs it +# there). Without one, it uses the npm package. # # WASM_OPT_NPM=1 always uses the npm package. set -euo pipefail @@ -31,7 +32,10 @@ native_wasm_opt() { echo "$bin" return fi - done < <(type -ap wasm-opt) + done < <( + type -ap wasm-opt || true + echo "$PWD/target/binaryen/binaryen-version_$VERSION/bin/wasm-opt" + ) } NATIVE="" @@ -44,7 +48,7 @@ if [ -n "$NATIVE" ]; then "$NATIVE" "${FLAGS[@]}" "$1" -o "$2" else if [ "${WASM_OPT_NPM:-}" != "1" ]; then - echo " native wasm-opt version_$VERSION not found, using the npm package (slower)" + echo " native wasm-opt version_$VERSION not found, using the npm package (slower, run \`pnpm setup\`)" fi pnpm exec wasm-opt "${FLAGS[@]}" "$1" -o "$2" fi From 1521a45afe9801cfb467be59bd0e27153558109a Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 01:39:16 -0300 Subject: [PATCH 04/12] build: reuse the wasm-opt output when its input did not change bin/wasm-opt.sh keeps the last output in target/wasm-opt, keyed by the input, the binaryen version and the script itself. A rebuild that does not change the Rust code skips wasm-opt. Co-Authored-By: Claude Opus 5.5 --- bin/wasm-opt.sh | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/bin/wasm-opt.sh b/bin/wasm-opt.sh index ec494cf..bac8d28 100755 --- a/bin/wasm-opt.sh +++ b/bin/wasm-opt.sh @@ -7,9 +7,13 @@ # It looks for it in PATH and in target/binaryen (`pnpm setup` installs it # there). Without one, it uses the npm package. # -# WASM_OPT_NPM=1 always uses the npm package. +# The last output is kept in target/wasm-opt and reused while the input does +# not change. +# +# WASM_OPT_NPM=1 always runs the npm package, without the cache. set -euo pipefail +SELF="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")" cd "$(dirname "$0")/.." FLAGS=( @@ -38,6 +42,21 @@ native_wasm_opt() { ) } +sha256() { + if command -v sha256sum > /dev/null; then sha256sum; else shasum -a 256; fi | cut -d' ' -f1 +} + +# The output only depends on the input, the binaryen version and the flags in +# this file: reuse the last output when none of them changed +CACHE_DIR=target/wasm-opt +CACHE="$CACHE_DIR/$({ sha256 < "$1"; echo "$VERSION"; sha256 < "$SELF"; } | sha256).wasm" + +if [ "${WASM_OPT_NPM:-}" != "1" ] && [ -f "$CACHE" ]; then + echo " input did not change, reusing $CACHE" + cp "$CACHE" "$2" + exit 0 +fi + NATIVE="" if [ "${WASM_OPT_NPM:-}" != "1" ]; then NATIVE=$(native_wasm_opt) @@ -52,3 +71,9 @@ else fi pnpm exec wasm-opt "${FLAGS[@]}" "$1" -o "$2" fi + +if [ "${WASM_OPT_NPM:-}" != "1" ]; then + rm -rf "$CACHE_DIR" + mkdir -p "$CACHE_DIR" + cp "$2" "$CACHE" +fi From 3e6a9de92a32e95d4033032fb2d684d2b9380350 Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 01:40:40 -0300 Subject: [PATCH 05/12] build: run tsdown while wasm-opt runs tsdown only reads js/ and never bundles lib/rregex.wasm, so both steps run in parallel. Each step still reports its own time; the total is the wall time. Co-Authored-By: Claude Opus 5.5 --- bin/bench-compare.mjs | 5 +++- bin/build.sh | 68 +++++++++++++++++++++++++------------------ 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/bin/bench-compare.mjs b/bin/bench-compare.mjs index 334ed27..26abdda 100644 --- a/bin/bench-compare.mjs +++ b/bin/bench-compare.mjs @@ -224,7 +224,10 @@ lines.push( 'RegExp is the time of the equivalent native `RegExp` call in the same runtime ' + '(`new` has no equivalent, `RegExp` compiles on first use). ' + 'ยฑ is the worst relative margin of error seen across rounds. ' + - (build ? 'The build time of `cargo build` depends on the state of the CI cache. ' : '') + + (build + ? 'The build time of `cargo build` depends on the state of the CI cache; ' + + '`wasm-opt` and `tsdown` run in parallel, so the total is less than the sum. ' + : '') + 'Shared CI runners are noisy: re-run the job before you act on a small regression.', ); diff --git a/bin/build.sh b/bin/build.sh index 659c7c4..dc53922 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -1,6 +1,7 @@ #!/bin/bash # Builds lib/. Prints the time of each step. With BUILD_TIMINGS=, it also # writes them to as JSON: { "steps": [{ "name", "ms" }], "total": ms }. +# wasm-opt and tsdown run in parallel, so the total is less than the sum. set -euo pipefail cd "$(dirname "$0")/.." @@ -18,50 +19,59 @@ now_ms() { esac } -STEPS=() -STARTED=$(now_ms) -STEP_NAME="" -STEP_START=0 +seconds() { + echo "$(($1 / 1000)).$(printf '%03d' $(($1 % 1000)))s" +} -step() { - end_step - echo " $1" - STEP_NAME=$2 - STEP_START=$(now_ms) +mkdir -p target +TIMES=target/build-times.txt +: > "$TIMES" + +# timed : runs the command and records its time +timed() { + local name=$1 start ms + shift + start=$(now_ms) + "$@" + ms=$(($(now_ms) - start)) + echo "$name $ms" >> "$TIMES" + echo " โฑ๏ธ $name: $(seconds "$ms")" } -end_step() { - [ -n "$STEP_NAME" ] || return 0 - local ms=$(($(now_ms) - STEP_START)) - STEPS+=("{\"name\":\"$STEP_NAME\",\"ms\":$ms}") - echo " โฑ๏ธ $STEP_NAME: $((ms / 1000)).$(printf '%03d' $((ms % 1000)))s" - STEP_NAME="" +bindings() { + rm -rf js/wasm lib + wasm-bindgen --target web --omit-default-module-path \ + --out-dir js/wasm --out-name rregex "$WASM" + node bin/metadata.mjs + mkdir -p lib } -step "๐Ÿฆ€ Compiling wasm..." "cargo build" -cargo build --release --target "$TARGET" +STARTED=$(now_ms) -step "๐Ÿ”— Generating bindings..." "wasm-bindgen" -rm -rf js/wasm lib -wasm-bindgen --target web --omit-default-module-path \ - --out-dir js/wasm --out-name rregex "$WASM" -node bin/metadata.mjs +echo " ๐Ÿฆ€ Compiling wasm..." +timed "cargo build" cargo build --release --target "$TARGET" -step "๐Ÿ—œ๏ธ Optimizing wasm..." "wasm-opt" -mkdir -p lib -./bin/wasm-opt.sh js/wasm/rregex_bg.wasm lib/rregex.wasm +echo " ๐Ÿ”— Generating bindings..." +timed "wasm-bindgen" bindings -step "๐Ÿ“ฆ Bundling..." "tsdown" -pnpm exec tsdown --silent +# tsdown only reads js/: lib/rregex.wasm is never bundled +echo " ๐Ÿ—œ๏ธ Optimizing wasm and ๐Ÿ“ฆ bundling..." +timed "wasm-opt" ./bin/wasm-opt.sh js/wasm/rregex_bg.wasm lib/rregex.wasm & +WASM_OPT=$! +timed "tsdown" pnpm exec tsdown --silent +wait "$WASM_OPT" -end_step TOTAL=$(($(now_ms) - STARTED)) if [ -n "${BUILD_TIMINGS:-}" ]; then + STEPS=() + for name in "cargo build" "wasm-bindgen" "wasm-opt" "tsdown"; do + STEPS+=("{\"name\":\"$name\",\"ms\":$(grep "^$name " "$TIMES" | awk '{print $NF}')}") + done ( IFS=, echo "{\"steps\":[${STEPS[*]}],\"total\":$TOTAL}" ) > "$BUILD_TIMINGS" fi -echo " โœ… Done in $((TOTAL / 1000)).$(printf '%03d' $((TOTAL % 1000)))s!" +echo " โœ… Done in $(seconds "$TOTAL")!" From a35e502eb1f567ad4bcf38128c7e7fa058ee9d5d Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 01:41:29 -0300 Subject: [PATCH 06/12] ci: run the Rust tests in their own job and use Swatinem/rust-cache - `test:rs` builds the crate in the test profile before the build. It now runs in the `test_rs` job, next to the build; `publish` still needs it. - The cargo cache key was only Cargo.lock: a new Rust on the runner restored a stale target/ and an exact hit never saved a new one. Swatinem/rust-cache adds the rustc version to the key and keeps only the dependencies. The release build and the benchmark share the key. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/benchmark.yml | 13 +++------- .github/workflows/release.yml | 45 +++++++++++++++++++++++++-------- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8686191..629f7a3 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -43,16 +43,11 @@ jobs: steps: - uses: actions/checkout@v7 - # same paths and key as the release workflow, so the cache of master is reused - - uses: actions/cache@v6 + # same shared key as the build of the release workflow, so the cache of + # master is reused + - uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- + shared-key: release # wasm-bindgen-cli must match the wasm-bindgen crate in Cargo.lock - name: reading wasm-bindgen version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 06fa6d7..e1e39c0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,15 +57,12 @@ jobs: registry-url: https://registry.npmjs.org/ cache: pnpm - - uses: actions/cache@v6 + # cargo registry and dependencies in target/. The key includes the rustc + # version, so a new Rust on the runner does not restore a stale cache. + # The benchmark workflow uses the same shared key. + - uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- + shared-key: release # wasm-bindgen-cli must match the wasm-bindgen crate in Cargo.lock - name: reading wasm-bindgen version @@ -86,9 +83,6 @@ jobs: - name: setup run: node --run setup - - name: testing wasm - run: node --run test:rs -- --node - - name: building run: node --run build @@ -97,6 +91,34 @@ jobs: name: lib path: lib/ + # the Rust tests build the crate in the test profile: in their own job, so + # the build and the jobs that need `lib` do not wait for them + test_rs: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: "26" + + - uses: Swatinem/rust-cache@v2 + + # wasm-bindgen-cli must match the wasm-bindgen crate in Cargo.lock + - name: reading wasm-bindgen version + id: wasm-bindgen + run: echo "version=$(grep -A1 '^name = "wasm-bindgen"$' Cargo.lock | sed -n 's/^version = "\(.*\)"$/\1/p')" >> "$GITHUB_OUTPUT" + + - uses: taiki-e/install-action@v2 + with: + tool: wasm-pack,wasm-bindgen@${{ steps.wasm-bindgen.outputs.version }} + + - run: rustup target add wasm32-unknown-unknown + + - name: testing wasm + run: node --run test:rs -- --node + test_node: runs-on: ubuntu-24.04 @@ -233,6 +255,7 @@ jobs: id-token: write # to enable use of OIDC for npm provenance needs: + - test_rs - test_node - test_bun - test_deno From 9c21efd1ca2cc178ff11fda6d649c7a99916c2e3 Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 02:34:24 -0300 Subject: [PATCH 07/12] build: generate the declarations that are the same as core.d.mts once core (cjs), commonjs and cf export exactly what core exports, so tsdown generated the same .d.ts four times. bin/build.sh now copies core.d.mts to the other three; lib/ is byte for byte the same. Co-Authored-By: Claude Opus 5.5 --- bin/build.sh | 10 +++++++++- tsdown.config.ts | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/bin/build.sh b/bin/build.sh index dc53922..3ca2c70 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -46,6 +46,14 @@ bindings() { mkdir -p lib } +# tsdown.config.ts does not generate these declarations: same as core.d.mts +bundle() { + pnpm exec tsdown --silent + for dts in core.d.cts commonjs.d.cts cf.d.mts; do + cp lib/core.d.mts "lib/$dts" + done +} + STARTED=$(now_ms) echo " ๐Ÿฆ€ Compiling wasm..." @@ -58,7 +66,7 @@ timed "wasm-bindgen" bindings echo " ๐Ÿ—œ๏ธ Optimizing wasm and ๐Ÿ“ฆ bundling..." timed "wasm-opt" ./bin/wasm-opt.sh js/wasm/rregex_bg.wasm lib/rregex.wasm & WASM_OPT=$! -timed "tsdown" pnpm exec tsdown --silent +timed "tsdown" bundle wait "$WASM_OPT" TOTAL=$(($(now_ms) - STARTED)) diff --git a/tsdown.config.ts b/tsdown.config.ts index 9a062bc..9e325a0 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -2,6 +2,10 @@ import { defineConfig, type UserConfig } from "tsdown"; // Every entry is one self-contained file in lib/ next to lib/rregex.wasm. // bin/build.sh generates js/wasm/ (wasm-bindgen glue and metadata) first. +// +// commonjs, core (cjs) and cf export exactly what core exports, so their +// declarations are the same as core.d.mts: bin/build.sh copies it instead of +// generating them (`dts: false`). const shared = { outDir: "lib", clean: false, @@ -29,6 +33,7 @@ export default defineConfig([ entry: { commonjs: "js/node.ts" }, platform: "node", format: "cjs", + dts: false, outExtensions: ext(".cjs", ".d.cts"), }, // Any runtime, the user loads the module @@ -44,6 +49,7 @@ export default defineConfig([ entry: { core: "js/core.ts" }, platform: "neutral", format: "cjs", + dts: false, outExtensions: ext(".cjs", ".d.cts"), }, // Browsers and bundlers @@ -60,6 +66,7 @@ export default defineConfig([ entry: { cf: "js/workerd.ts" }, platform: "neutral", format: "esm", + dts: false, deps: { onlyBundle: false, neverBundle: [/\.wasm$/] }, outExtensions: ext(".mjs", ".d.mts"), }, From 6571118fc12c3bdd1e808683f806761676e50fa5 Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 02:34:52 -0300 Subject: [PATCH 08/12] ci: cache the native binaryen and check the regex engine versions in their own job - rust-cache does not keep target/binaryen, so `setup` downloaded it on every run; actions/cache keeps it, keyed by the binaryen version. - bin/check-regex-deps.sh asks crates.io for publish dates before the build. It now runs in the `check_deps` job, next to the build; `publish` needs it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/benchmark.yml | 17 ++++++++++++++--- .github/workflows/release.yml | 29 ++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 629f7a3..57abdeb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -81,10 +81,21 @@ jobs: test -f .latest/lib/esm.mjs -a -f .latest/lib/rregex.wasm echo "LATEST_VERSION=$(jq -r .version <<< "$meta")" >> "$GITHUB_ENV" + - name: installing + run: pnpm install --frozen-lockfile + + # native binaryen for bin/wasm-opt.sh; rust-cache does not keep it + - name: reading binaryen version + id: binaryen + run: echo "version=$(node -p 'require("./node_modules/binaryen/package.json").version')" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v6 + with: + path: target/binaryen + key: ${{ runner.os }}-binaryen-${{ steps.binaryen.outputs.version }} + - name: setup - run: | - pnpm install --frozen-lockfile - node --run setup + run: node --run setup # check that the lib is complete; the build time goes to the report - name: building head diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e1e39c0..33e3858 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,12 +74,19 @@ jobs: with: tool: wasm-pack,wasm-bindgen@${{ steps.wasm-bindgen.outputs.version }} - - name: checking regex engine versions - run: ./bin/check-regex-deps.sh - - name: installing run: pnpm install --frozen-lockfile + # native binaryen for bin/wasm-opt.sh; rust-cache does not keep it + - name: reading binaryen version + id: binaryen + run: echo "version=$(node -p 'require("./node_modules/binaryen/package.json").version')" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v6 + with: + path: target/binaryen + key: ${{ runner.os }}-binaryen-${{ steps.binaryen.outputs.version }} + - name: setup run: node --run setup @@ -91,6 +98,21 @@ jobs: name: lib path: lib/ + # asks crates.io for the publish dates: in its own job, next to the build + check_deps: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v7 + with: + sparse-checkout: | + Cargo.lock + bin/check-regex-deps.sh + sparse-checkout-cone-mode: false + + - name: checking regex engine versions + run: ./bin/check-regex-deps.sh + # the Rust tests build the crate in the test profile: in their own job, so # the build and the jobs that need `lib` do not wait for them test_rs: @@ -255,6 +277,7 @@ jobs: id-token: write # to enable use of OIDC for npm provenance needs: + - check_deps - test_rs - test_node - test_bun From e2e1ae76700a23c30b2625bd769cc86a146a211c Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 02:36:13 -0300 Subject: [PATCH 09/12] build: install wasm-bindgen-cli and wasm-pack with cargo-binstall when available `cargo install` compiles each one in ~1-2 min; cargo-binstall downloads the prebuilt binary. Without it, bin/install.sh still uses `cargo install`. Co-Authored-By: Claude Opus 5.5 --- bin/install.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index 0417c06..5e655d2 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -17,13 +17,23 @@ echo "" rustup target add wasm32-unknown-unknown +## prebuilt binaries with cargo-binstall when it is installed; +## `cargo install` compiles each one (~1-2 min) +cargo_install() { + if cargo binstall -V > /dev/null 2>&1; then + cargo binstall --no-confirm "$@" + else + cargo install "$@" + fi +} + ## wasm-bindgen-cli must match the wasm-bindgen crate version in Cargo.lock WASM_BINDGEN_REQUIRED=$(grep -A1 '^name = "wasm-bindgen"$' Cargo.lock | sed -n 's/^version = "\(.*\)"$/\1/p') WASM_BINDGEN_VERSION=$(wasm-bindgen --version 2> /dev/null | cut -d' ' -f2 || true) if [ "$WASM_BINDGEN_VERSION" != "$WASM_BINDGEN_REQUIRED" ]; then echo " ๐Ÿ“ฅ installing wasm-bindgen-cli $WASM_BINDGEN_REQUIRED" - cargo install wasm-bindgen-cli --version "$WASM_BINDGEN_REQUIRED" --locked --force + cargo_install wasm-bindgen-cli --version "$WASM_BINDGEN_REQUIRED" --locked --force fi echo " โœ… wasm-bindgen $(wasm-bindgen --version | cut -d' ' -f2) ($(which wasm-bindgen))" @@ -34,7 +44,7 @@ WASM_PACK_VERSION=$(wasm-pack --version 2> /dev/null || true) ## wasm-pack is only used to run the Rust tests (pnpm test:rs) if [ "$WASM_PACK_VERSION" = "" ]; then echo " ๐Ÿ“ฅ installing wasm-pack" - cargo install wasm-pack + cargo_install wasm-pack WASM_PACK_VERSION=$(wasm-pack --version) fi From 2e3d443067873d6d4062ef517f3f54e2b98648ba Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 02:42:30 -0300 Subject: [PATCH 10/12] ci: run each benchmark runtime in its own job The benchmark ran Node, Chrome and Firefox one after the other in one job: ~5 min of a 5:31 job. Now `build` builds head once and downloads base, `bench` runs one job per runtime in parallel, and `report` writes the comment from their samples. Every runtime measures the same two builds, and every delta still compares base and head on the same runner. - bin/bench-node.mjs: the Node measurement of bench-compare.mjs, with the same JSON output as bench-browser.mjs. - bin/bench-compare.mjs: only the report, from one or more --runtime files; it shows the CPU model of each runtime. - bin/bench-browser.mjs: the 5 s timer of the teardown kept the process alive after the browser exited (~5 s between runtimes). It is now unref'ed and cleared, the browser gets SIGKILL once the results are in, and the log shows the start and close times of the browser. - The benchmark build does not save the rust cache: with the release build it failed to reserve the same key. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/benchmark.yml | 142 +++++++++++++++++++++++++++----- README.md | 9 +- bin/bench-browser.mjs | 28 +++++-- bin/bench-cases.mjs | 2 +- bin/bench-compare.mjs | 123 +++++++-------------------- bin/bench-node.mjs | 101 +++++++++++++++++++++++ test/browser/bench.html | 2 +- test/browser/compare.mjs | 2 +- test/browser/compare.worker.mjs | 2 +- 9 files changed, 283 insertions(+), 128 deletions(-) mode change 100644 => 100755 bin/bench-compare.mjs create mode 100755 bin/bench-node.mjs diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 57abdeb..827595f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -33,21 +33,25 @@ permissions: contents: read jobs: - benchmark: + # builds head once and downloads base, so every runtime measures the same + # two builds + build: runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 15 - env: - HEAD_SHA: ${{ github.event.pull_request.head.sha }} + outputs: + latest: ${{ steps.latest.outputs.version }} steps: - uses: actions/checkout@v7 # same shared key as the build of the release workflow, so the cache of - # master is reused + # master is reused. Only the release workflow saves it: two jobs that + # save the same key at the same time fail to reserve it. - uses: Swatinem/rust-cache@v2 with: shared-key: release + save-if: false # wasm-bindgen-cli must match the wasm-bindgen crate in Cargo.lock - name: reading wasm-bindgen version @@ -72,6 +76,7 @@ jobs: # the base is the published package: inside the repository, because the # browser benchmark only serves files from there - name: downloading latest + id: latest run: | meta=$(curl -fsSL https://registry.npmjs.org/rregex/latest) curl -fsSL -o "$RUNNER_TEMP/rregex.tgz" "$(jq -r .dist.tarball <<< "$meta")" @@ -79,7 +84,7 @@ jobs: mkdir -p .latest tar xzf "$RUNNER_TEMP/rregex.tgz" -C .latest --strip-components=1 package/lib test -f .latest/lib/esm.mjs -a -f .latest/lib/rregex.wasm - echo "LATEST_VERSION=$(jq -r .version <<< "$meta")" >> "$GITHUB_ENV" + echo "version=$(jq -r .version <<< "$meta")" >> "$GITHUB_OUTPUT" - name: installing run: pnpm install --frozen-lockfile @@ -100,29 +105,119 @@ jobs: # check that the lib is complete; the build time goes to the report - name: building head run: | - mkdir -p "$RUNNER_TEMP/benchmark" - BUILD_TIMINGS="$RUNNER_TEMP/benchmark/build.json" node --run build + BUILD_TIMINGS=build.json node --run build test -f lib/esm.mjs -a -f lib/rregex.wasm - # Chrome and Firefox come with the runner image; the runtimes run one - # after the other, so they do not compete for the CPU + - uses: actions/upload-artifact@v7 + with: + name: bench-libs + path: | + lib/ + .latest/lib/ + build.json + include-hidden-files: true # .latest + if-no-files-found: error + retention-days: 1 + + # One job for each runtime, each on its own runner, so they run in parallel + # and do not compete for the CPU. Every delta compares base and head on the + # same runner, so the runtimes do not need the same hardware. + bench: + needs: build + + runs-on: ubuntu-24.04 + timeout-minutes: 20 + + strategy: + matrix: + runtime: [node, chrome, firefox] + + steps: + - uses: actions/checkout@v7 + + # tinybench, for the Node benchmark only + - uses: pnpm/action-setup@v6 + if: matrix.runtime == 'node' + with: + version: 11 + + # also the runtime of the Node benchmark + - name: Run actions/setup-node@v7 + uses: actions/setup-node@v7 + with: + node-version: "26" + cache: ${{ matrix.runtime == 'node' && 'pnpm' || '' }} + + - uses: actions/download-artifact@v8 + with: + name: bench-libs + + - name: installing + if: matrix.runtime == 'node' + run: pnpm install --frozen-lockfile + + # Chrome and Firefox come with the runner image - name: running benchmarks + env: + RUNTIME: ${{ matrix.runtime }} + run: | + mkdir -p "$RUNNER_TEMP/benchmark" + if [ "$RUNTIME" = node ]; then + node bin/bench-node.mjs --base .latest/lib --head lib \ + --json "$RUNNER_TEMP/benchmark/$RUNTIME.json" + else + node bin/bench-browser.mjs --browser "$RUNTIME" --base .latest/lib --head lib \ + --json "$RUNNER_TEMP/benchmark/$RUNTIME.json" + fi + + - uses: actions/upload-artifact@v7 + with: + name: benchmark-${{ matrix.runtime }} + path: ${{ runner.temp }}/benchmark/${{ matrix.runtime }}.json + if-no-files-found: error + retention-days: 1 + + report: + needs: [build, bench] + + runs-on: ubuntu-24.04 + timeout-minutes: 5 + + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + LATEST_VERSION: ${{ needs.build.outputs.latest }} + + steps: + # bench-compare.mjs only needs bin/ and Node + - uses: actions/checkout@v7 + with: + sparse-checkout: bin + + - uses: actions/setup-node@v7 + with: + node-version: "26" + + - uses: actions/download-artifact@v8 + with: + name: bench-libs + + - uses: actions/download-artifact@v8 + with: + pattern: benchmark-* + merge-multiple: true + path: ${{ runner.temp }}/benchmark + + - name: writing report env: REPORT_DIR: ${{ runner.temp }}/benchmark run: | - mkdir -p "$REPORT_DIR" - for browser in chrome firefox; do - node bin/bench-browser.mjs \ - --browser "$browser" \ - --base .latest/lib \ - --head lib \ - --json "$REPORT_DIR/$browser.json" - done + cp build.json "$REPORT_DIR/build.json" node bin/bench-compare.mjs \ --base .latest/lib \ --head lib \ --base-label "\`rregex@${LATEST_VERSION}\` (npm latest)" \ --head-label "\`${HEAD_SHA::7}\`" \ + --runtime "$REPORT_DIR/node.json" \ --runtime "$REPORT_DIR/chrome.json" \ --runtime "$REPORT_DIR/firefox.json" \ --build-time "$REPORT_DIR/build.json" \ @@ -130,6 +225,7 @@ jobs: --json "$REPORT_DIR/results.json" cat "$REPORT_DIR/report.md" >> "$GITHUB_STEP_SUMMARY" + # the `benchmark comment` workflow downloads it by this name - uses: actions/upload-artifact@v7 with: name: benchmark @@ -139,11 +235,13 @@ jobs: comment: # PRs from this repository only, see `benchmark comment` for the others if: >- - !cancelled() && needs.benchmark.result != 'skipped' && + !cancelled() && needs.build.result != 'skipped' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login != 'dependabot[bot]' - needs: benchmark + # all of them: when build or bench fails, report is skipped and the + # comment reports the failure + needs: [build, bench, report] runs-on: ubuntu-24.04 @@ -157,14 +255,14 @@ jobs: sparse-checkout: .github/scripts - uses: actions/download-artifact@v8 - if: needs.benchmark.result == 'success' + if: needs.report.result == 'success' with: name: benchmark path: ${{ runner.temp }}/benchmark - uses: actions/github-script@v9 env: - SUCCESS: ${{ needs.benchmark.result == 'success' }} + SUCCESS: ${{ needs.report.result == 'success' }} with: script: | await require('./.github/scripts/benchmark-comment.cjs')({ diff --git a/README.md b/README.md index 785c544..608224e 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,14 @@ The time of `rregex` is predictable: the time to copy the input plus a time that ### Run the benchmarks - **Browser:** run `pnpm build`, then `pnpm test:browser`, and open . The page shows the charts above and gives a Markdown table and SVG/PNG images of the results. -- **Node:** the same cases are in [`bin/bench-cases.mjs`](./bin/bench-cases.mjs). [`bin/bench-compare.mjs`](./bin/bench-compare.mjs) compares two builds of `rregex` with each other, and CI uses it on each pull request. +- **Node:** the same cases are in [`bin/bench-cases.mjs`](./bin/bench-cases.mjs). +- **Compare two builds:** [`bin/bench-node.mjs`](./bin/bench-node.mjs) and [`bin/bench-browser.mjs`](./bin/bench-browser.mjs) run base and head side by side in one runtime and write the samples to a JSON file. [`bin/bench-compare.mjs`](./bin/bench-compare.mjs) makes the report from these files. CI does this on each pull request, with one job for each runtime: + + ```sh + node bin/bench-node.mjs --base --head lib --json node.json + node bin/bench-browser.mjs --browser chrome --base --head lib --json chrome.json + node bin/bench-compare.mjs --base --head lib --runtime node.json --runtime chrome.json + ``` ## Known Issues diff --git a/bin/bench-browser.mjs b/bin/bench-browser.mjs index c58ce49..2b2f8d8 100644 --- a/bin/bench-browser.mjs +++ b/bin/bench-browser.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Runs the base vs head benchmarks of bin/bench-compare.mjs in a browser and +// Runs the base vs head benchmarks of bin/bench-cases.mjs in a browser and // writes the samples to a JSON file, for `bench-compare.mjs --runtime `. // // Usage: @@ -16,7 +16,7 @@ import { spawn } from 'node:child_process'; import { mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; +import { cpus, tmpdir } from 'node:os'; import { join, relative, resolve, sep } from 'node:path'; import { parseArgs } from 'node:util'; import { ROOT, createStaticServer } from '../test/browser/server.mjs'; @@ -67,10 +67,19 @@ function urlPath(dir) { return `/${path.split(sep).join('/')}`; } +const started = performance.now(); +const elapsed = () => `${((performance.now() - started) / 1000).toFixed(1)} s`; +let ready = false; + const results = Promise.withResolvers(); const server = createStaticServer({ onPost(pathname, message) { if (pathname !== '/results') return; + // the page posts its first message once the browser runs it + if (!ready) { + ready = true; + console.error(`${args.browser}: page ready after ${elapsed()}`); + } if (message.type === 'progress') console.error(message.message); if (message.type === 'done') results.resolve(message.results); if (message.type === 'error') results.reject(new Error(message.message)); @@ -110,7 +119,7 @@ const timer = setTimeout( ); try { - const json = JSON.stringify(await results.promise, null, 2) + '\n'; + const json = JSON.stringify({ ...(await results.promise), cpu: cpus()[0]?.model }, null, 2) + '\n'; if (args.json) { await writeFile(args.json, json); } else { @@ -121,18 +130,25 @@ try { process.exitCode = 1; } finally { clearTimeout(timer); + // SIGKILL: the results are in (or the run failed), so nothing in the + // browser needs a clean shutdown, and Firefox can take seconds on SIGTERM if (browser && browser.exitCode === null && browser.signalCode === null) { + const closing = performance.now(); browser.removeAllListeners('exit'); + let timeout; const exited = new Promise((done) => { browser.once('exit', done); - setTimeout(done, 5000); + // unref: a pending timer would keep the process alive after the exit + timeout = setTimeout(done, 5000).unref(); }); try { - process.kill(-browser.pid, 'SIGTERM'); + process.kill(-browser.pid, 'SIGKILL'); } catch { - browser.kill('SIGTERM'); + browser.kill('SIGKILL'); } await exited; + clearTimeout(timeout); + console.error(`${args.browser}: closed in ${((performance.now() - closing) / 1000).toFixed(1)} s`); } server.closeAllConnections(); server.close(); diff --git a/bin/bench-cases.mjs b/bin/bench-cases.mjs index 60ed7b2..a604666 100644 --- a/bin/bench-cases.mjs +++ b/bin/bench-cases.mjs @@ -1,4 +1,4 @@ -// Benchmark cases shared by bin/bench-compare.mjs (Node) and +// Benchmark cases shared by bin/bench-node.mjs (Node) and // test/browser/compare.worker.mjs (browsers), so every runtime runs the // same calls on the same input. diff --git a/bin/bench-compare.mjs b/bin/bench-compare.mjs old mode 100644 new mode 100755 index 26abdda..8cab633 --- a/bin/bench-compare.mjs +++ b/bin/bench-compare.mjs @@ -1,70 +1,25 @@ #!/usr/bin/env node -// Compares the performance of two rregex builds (base vs head), and of head -// vs the native RegExp, and writes a Markdown report and a JSON file with the -// results. +// Writes the report of the benchmarks: compares two rregex builds (base vs +// head) in each runtime, and head vs the native RegExp, as Markdown and JSON. // // Usage: // node bin/bench-compare.mjs --base --head \ -// [--base-label ] [--head-label ] \ -// [--runtime ]... [--build-time ] \ -// [--markdown ] [--json ] \ -// [--rounds 5] [--time 250] [--threshold 10] +// --runtime [--runtime ]... \ +// [--base-label ] [--head-label ] [--build-time ] \ +// [--markdown ] [--json ] [--threshold 10] // -// The cases run in Node. Each --runtime adds the results of another runtime, -// written by bin/bench-browser.mjs, to the report. --build-time adds the build -// time of head, written by bin/build.sh (BUILD_TIMINGS=). +// Each --runtime is the samples of one runtime, written by bin/bench-node.mjs +// or bin/bench-browser.mjs. The runtimes can run on different machines: every +// delta compares base and head on the same machine. --build-time adds the +// build time of head, written by bin/build.sh (BUILD_TIMINGS=). The lib +// dirs only give the size of the wasm files. // -// To reduce noise, base and head run next to each other for every case, the -// order alternates on each round, and the reported delta is the median of the -// per-round head/base ratios. -// -// Every (round, case) pair runs in a new worker thread. This gives each case a -// clean wasm memory: older builds use the `wee_alloc` allocator, where calls -// with large inputs grow the wasm memory until it hits the 4 GiB limit and trap. +// The reported delta is the median of the per-round head/base ratios. -import { Bench } from 'tinybench'; import { parseArgs } from 'node:util'; import { readFile, writeFile, stat } from 'node:fs/promises'; -import { resolve, dirname } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; -import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads'; -import { - DATA_PATH, - addRound, - cases, - emptySamples, - nativeCases, - roundOrder, - sample, - summarize, -} from './bench-cases.mjs'; - -const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); -const data = sample(await readFile(resolve(root, DATA_PATH), 'utf8')); - -const importLib = (dir) => import(pathToFileURL(resolve(dir, 'esm.mjs')).href); - -if (!isMainThread) { - // Worker: run one case for both builds and the native RegExp, in the given order. - const { dirs, name, order, time } = workerData; - const bench = new Bench({ time, warmupTime: Math.min(100, time) }); - for (const build of order) { - const fn = - build === 'native' - ? nativeCases(data).get(name) - : cases(await importLib(dirs[build]), data).get(name); - if (fn) bench.add(build, fn); - } - await bench.warmup(); - await bench.run(); - const result = {}; - for (const task of bench.tasks) { - if (task.result?.error) throw task.result.error; - result[task.name] = { mean: task.result.mean, rme: task.result.rme }; - } - parentPort.postMessage(result); - process.exit(0); -} +import { resolve } from 'node:path'; +import { summarize } from './bench-cases.mjs'; const { values: args } = parseArgs({ options: { @@ -76,19 +31,15 @@ const { values: args } = parseArgs({ 'build-time': { type: 'string' }, markdown: { type: 'string' }, json: { type: 'string' }, - rounds: { type: 'string', default: process.env.BENCH_ROUNDS ?? '5' }, - time: { type: 'string', default: process.env.BENCH_TIME ?? '250' }, threshold: { type: 'string', default: process.env.BENCH_THRESHOLD ?? '10' }, }, }); -if (!args.base || !args.head) { - console.error('Usage: bench-compare.mjs --base --head [options]'); +if (!args.base || !args.head || !args.runtime.length) { + console.error('Usage: bench-compare.mjs --base --head --runtime ... [options]'); process.exit(2); } -const ROUNDS = Number(args.rounds); -const TIME = Number(args.time); const THRESHOLD = Number(args.threshold) / 100; const dirs = { base: resolve(args.base), head: resolve(args.head) }; @@ -100,36 +51,18 @@ size.delta = size.head / size.base - 1; const build = args['build-time'] ? JSON.parse(await readFile(args['build-time'], 'utf8')) : null; -const names = [...cases(await importLib(dirs.head), data).keys()]; - -/** Runs one case for both builds in a new worker thread. */ -const runCase = (name, order) => - new Promise((done, fail) => { - const worker = new Worker(fileURLToPath(import.meta.url), { - workerData: { dirs, name, order, time: TIME }, - }); - worker.once('message', done); - worker.once('error', fail); - worker.once('exit', (code) => code && fail(new Error(`${name}: worker exited with code ${code}`))); - }); - -const samples = Object.fromEntries(names.map((name) => [name, emptySamples()])); - -for (let round = 0; round < ROUNDS; round++) { - for (const name of names) { - addRound(samples[name], await runCase(name, roundOrder(round))); - } - console.error(`round ${round + 1}/${ROUNDS} done`); -} - -const runtimes = [ - { runtime: `Node ${process.version}`, rounds: ROUNDS, time: TIME, results: summarize(samples, THRESHOLD) }, -]; +const runtimes = []; for (const file of args.runtime) { - const other = JSON.parse(await readFile(file, 'utf8')); - runtimes.push({ ...other, samples: undefined, results: summarize(other.samples, THRESHOLD) }); + const { samples, ...runtime } = JSON.parse(await readFile(file, 'utf8')); + runtimes.push({ ...runtime, results: summarize(samples, THRESHOLD) }); } +// every case of every runtime, in the order of the first one that has it +const names = [...new Set(runtimes.flatMap(({ results }) => results.map((r) => r.name)))]; + +// "5 rounds ร— 250 ms", once if every runtime used the same settings +const settings = [...new Set(runtimes.map((r) => `${r.rounds} rounds ร— ${r.time} ms`))]; + const regressions = runtimes.flatMap(({ runtime, results }) => results.filter((r) => r.status === 'regression').map((r) => ({ runtime, ...r })), ); @@ -154,7 +87,7 @@ const lines = []; lines.push('## โฑ๏ธ Benchmark results', ''); lines.push( `Comparing **head** ${args['head-label']} against **base** ${args['base-label']} ` + - `on ${list(runtimes.map((r) => r.runtime))} (${ROUNDS} rounds ร— ${TIME} ms per case). ` + + `on ${list(runtimes.map((r) => r.runtime))} (${settings.join(' / ')} per case). ` + `Changes within ยฑ${args.threshold}% are treated as noise.`, '', ); @@ -190,8 +123,8 @@ for (const name of names) { lines.push(''); // details: the times of each runtime -for (const { runtime, results } of runtimes) { - lines.push(`
${runtime}: times`, ''); +for (const { runtime, cpu, results } of runtimes) { + lines.push(`
${runtime}: times${cpu ? ` (${cpu})` : ''}`, ''); lines.push('| | Benchmark | base | head | RegExp | ฮ” time | ยฑ |'); lines.push('|:-:|:--|--:|--:|--:|--:|--:|'); for (const r of results) { @@ -232,7 +165,7 @@ lines.push( ); const markdown = lines.join('\n') + '\n'; -const json = { rounds: ROUNDS, time: TIME, threshold: THRESHOLD, size, build, runtimes }; +const json = { threshold: THRESHOLD, size, build, runtimes }; if (args.markdown) await writeFile(args.markdown, markdown); if (args.json) await writeFile(args.json, JSON.stringify(json, null, 2) + '\n'); diff --git a/bin/bench-node.mjs b/bin/bench-node.mjs new file mode 100755 index 0000000..b62ea57 --- /dev/null +++ b/bin/bench-node.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node +// Runs the base vs head benchmarks in Node and writes the samples to a JSON +// file, for `bench-compare.mjs --runtime ` (same format as +// bin/bench-browser.mjs). +// +// Usage: +// node bin/bench-node.mjs --base --head \ +// [--json ] [--rounds 5] [--time 250] +// +// To reduce noise, base and head run next to each other for every case and +// the order alternates on each round (see bin/bench-cases.mjs). +// +// Every (round, case) pair runs in a new worker thread. This gives each case a +// clean wasm memory: older builds use the `wee_alloc` allocator, where calls +// with large inputs grow the wasm memory until it hits the 4 GiB limit and trap. + +import { Bench } from 'tinybench'; +import { parseArgs } from 'node:util'; +import { readFile, writeFile } from 'node:fs/promises'; +import { cpus } from 'node:os'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads'; +import { DATA_PATH, addRound, cases, emptySamples, nativeCases, roundOrder, sample } from './bench-cases.mjs'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const data = sample(await readFile(resolve(root, DATA_PATH), 'utf8')); + +const importLib = (dir) => import(pathToFileURL(resolve(dir, 'esm.mjs')).href); + +if (!isMainThread) { + // Worker: run one case for both builds and the native RegExp, in the given order. + const { dirs, name, order, time } = workerData; + const bench = new Bench({ time, warmupTime: Math.min(100, time) }); + for (const build of order) { + const fn = + build === 'native' + ? nativeCases(data).get(name) + : cases(await importLib(dirs[build]), data).get(name); + if (fn) bench.add(build, fn); + } + await bench.warmup(); + await bench.run(); + const result = {}; + for (const task of bench.tasks) { + if (task.result?.error) throw task.result.error; + result[task.name] = { mean: task.result.mean, rme: task.result.rme }; + } + parentPort.postMessage(result); + process.exit(0); +} + +const { values: args } = parseArgs({ + options: { + base: { type: 'string' }, + head: { type: 'string' }, + json: { type: 'string' }, + rounds: { type: 'string', default: process.env.BENCH_ROUNDS ?? '5' }, + time: { type: 'string', default: process.env.BENCH_TIME ?? '250' }, + }, +}); + +if (!args.base || !args.head) { + console.error('Usage: bench-node.mjs --base --head [--json ] [options]'); + process.exit(2); +} + +const rounds = Number(args.rounds); +const time = Number(args.time); +const runtime = `Node ${process.version}`; +const dirs = { base: resolve(args.base), head: resolve(args.head) }; +const names = [...cases(await importLib(dirs.head), data).keys()]; + +/** Runs one case for both builds in a new worker thread. */ +const runCase = (name, order) => + new Promise((done, fail) => { + const worker = new Worker(fileURLToPath(import.meta.url), { + workerData: { dirs, name, order, time }, + }); + worker.once('message', done); + worker.once('error', fail); + worker.once('exit', (code) => code && fail(new Error(`${name}: worker exited with code ${code}`))); + }); + +console.error(`${runtime}: ${names.length} cases ร— ${rounds} rounds ร— ${time} ms`); + +const samples = Object.fromEntries(names.map((name) => [name, emptySamples()])); +for (let round = 0; round < rounds; round++) { + for (const name of names) { + addRound(samples[name], await runCase(name, roundOrder(round))); + } + console.error(`${runtime}: round ${round + 1}/${rounds} done`); +} + +const results = { runtime, cpu: cpus()[0]?.model, rounds, time, samples }; +const json = JSON.stringify(results, null, 2) + '\n'; +if (args.json) { + await writeFile(args.json, json); +} else { + process.stdout.write(json); +} diff --git a/test/browser/bench.html b/test/browser/bench.html index 729931c..0e8e385 100644 --- a/test/browser/bench.html +++ b/test/browser/bench.html @@ -66,7 +66,7 @@

Same patterns and input (test/__data__/input-text.txt) as - bin/bench-compare.mjs. Both engines compile the pattern once, before the + bin/bench-cases.mjs. Both engines compile the pattern once, before the benchmark. \w is ASCII in RegExp and Unicode in RRegex. Hover a bar for its numbers.

diff --git a/test/browser/compare.mjs b/test/browser/compare.mjs index e349183..47a187e 100644 --- a/test/browser/compare.mjs +++ b/test/browser/compare.mjs @@ -1,4 +1,4 @@ -// Browser version of bin/bench-compare.mjs: runs every case for base, head +// Browser version of bin/bench-node.mjs: runs every case for base, head // and the native RegExp, and reports the samples to `?report=` (used by // bin/bench-browser.mjs) or as `globalThis.compareResults`. // diff --git a/test/browser/compare.worker.mjs b/test/browser/compare.worker.mjs index c2e6242..4b5dd66 100644 --- a/test/browser/compare.worker.mjs +++ b/test/browser/compare.worker.mjs @@ -1,6 +1,6 @@ // Runs one case for both builds and the native RegExp, in the given order. // Each case runs in a new worker, so every build starts with a clean wasm -// memory (see bin/bench-compare.mjs). +// memory (see bin/bench-node.mjs). import { cases, nativeCases } from "../../bin/bench-cases.mjs" import { measure } from "./measure.mjs" From 592f78549850b63116563d5f45e57a3292e607bc Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 02:42:37 -0300 Subject: [PATCH 11/12] ci: run every version of a runtime in one test job The tests of each Node, Bun and Deno version take seconds, less than the setup of a runner, but each version had its own job: 11 of the 14 jobs of a PR. With a few PRs open the jobs waited for runners (up to 21 min). Now each runtime runs its versions one after the other in one job; a failed version stops the job, like the fail-fast of the matrix, and each step names its version. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 124 +++++++++++++++++++++++----------- 1 file changed, 84 insertions(+), 40 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33e3858..b551243 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -141,89 +141,133 @@ jobs: - name: testing wasm run: node --run test:rs -- --node + # Each runtime runs every version in one job: the tests take seconds, less + # than the setup of a runner, and one job per version used ~11 of the + # concurrent jobs of the account on each PR. A failed version stops the job, + # like the fail-fast of a matrix. test_node: runs-on: ubuntu-24.04 needs: build - strategy: - matrix: - version: ["22", "24", "26"] - steps: - uses: actions/checkout@v7 - - name: Run actions/setup-node@v7 (${{ matrix.version }}) - uses: actions/setup-node@v7 - with: - node-version: ${{ matrix.version }} - registry-url: https://registry.npmjs.org/ - - uses: actions/download-artifact@v8 with: name: lib path: lib - - run: node lib/commonjs.cjs - - run: node lib/esm.mjs - - run: node --test test/node.test.cjs - - run: node --test test/node.test.mjs + - uses: actions/setup-node@v7 + with: + node-version: "22" + + - name: node 22 + run: &test-node | + node --version + node lib/commonjs.cjs + node lib/esm.mjs + node --test test/node.test.cjs + node --test test/node.test.mjs + + - uses: actions/setup-node@v7 + with: + node-version: "24" + + - name: node 24 + run: *test-node + + - uses: actions/setup-node@v7 + with: + node-version: "26" + + - name: node 26 + run: *test-node test_bun: runs-on: ubuntu-24.04 needs: build - strategy: - matrix: - version: ["1.0", "1.1", "1.2", "1.3", "1.4"] - steps: - uses: actions/checkout@v7 - - name: Run oven-sh/setup-bun@v2 (${{ matrix.version }}) - uses: oven-sh/setup-bun@v2 - with: - bun-version: ${{ matrix.version }} - - uses: actions/download-artifact@v8 with: name: lib path: lib - - run: bun lib/commonjs.cjs - - run: bun lib/esm.mjs - - run: bun test test/bun.test.cjs - - run: bun test test/bun.test.mjs + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.0" + + - name: bun 1.0 + run: &test-bun | + bun --version + bun lib/commonjs.cjs + bun lib/esm.mjs + bun test test/bun.test.cjs + bun test test/bun.test.mjs + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.1" + + - name: bun 1.1 + run: *test-bun + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.2" + + - name: bun 1.2 + run: *test-bun + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3" + + - name: bun 1.3 + run: *test-bun + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.4" + + - name: bun 1.4 + run: *test-bun test_deno: runs-on: ubuntu-24.04 needs: build - strategy: - matrix: - version: ["1.x", "2.x"] - steps: - uses: actions/checkout@v7 - - name: Run denoland/setup-deno@v2 (${{ matrix.version }}) - uses: denoland/setup-deno@v2 - with: - deno-version: "v${{ matrix.version }}" - - uses: actions/download-artifact@v8 with: name: lib path: lib - - run: deno install - if: ${{ matrix.version != '1.x' }} + - uses: denoland/setup-deno@v2 + with: + deno-version: v1.x + + - name: deno 1.x + run: &test-deno | + deno --version + deno run --allow-read=lib/rregex.wasm lib/esm.mjs + deno test --allow-read=lib/rregex.wasm test/deno.test.mjs - - run: deno run --allow-read=lib/rregex.wasm lib/esm.mjs + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + + - run: deno install - - run: deno test --allow-read=lib/rregex.wasm test/deno.test.mjs + - name: deno 2.x + run: *test-deno test_cf: runs-on: ubuntu-24.04 From a595b9f4a8495fe2039e298359ab811149cc17c4 Mon Sep 17 00:00:00 2001 From: frami Date: Sun, 1 Nov 2026 02:42:37 -0300 Subject: [PATCH 12/12] ci: report the changes in the public API on PRs The `api` job compares the type declarations in lib/ and the entry points of package.json (`exports`, `main`, `module`, `types`, `browser`) of the PR with the latest release on npm, and posts the diff as a PR comment. Files with the same change share one diff. The comment is created only when the API changes, and an earlier one is updated either way. It runs in the benchmark workflow, which builds head and downloads the latest release already, and whose paths cover every input of the build (now with tsconfig.json). The `benchmark comment` workflow posts it for PRs from forks; it reads only a boolean from the JSON of the PR run. .github/scripts/benchmark-comment.cjs is now pr-comment.cjs, with the marker and the title as options. Co-Authored-By: Claude Opus 5.5 --- .../{benchmark-comment.cjs => pr-comment.cjs} | 43 ++-- .github/workflows/benchmark-comment.yml | 52 +++-- .github/workflows/benchmark.yml | 121 +++++++++++- bin/api-diff.mjs | 187 ++++++++++++++++++ 4 files changed, 370 insertions(+), 33 deletions(-) rename .github/scripts/{benchmark-comment.cjs => pr-comment.cjs} (55%) create mode 100755 bin/api-diff.mjs diff --git a/.github/scripts/benchmark-comment.cjs b/.github/scripts/pr-comment.cjs similarity index 55% rename from .github/scripts/benchmark-comment.cjs rename to .github/scripts/pr-comment.cjs index 24c3a3f..0baae77 100644 --- a/.github/scripts/benchmark-comment.cjs +++ b/.github/scripts/pr-comment.cjs @@ -1,9 +1,9 @@ -// Creates or updates the benchmark comment on a PR. Used by actions/github-script -// from the `benchmark` workflow (PRs from this repository) and from the -// `benchmark comment` workflow (PRs from forks and dependabot). +// Creates or updates a report comment on a PR (one comment per marker, kept up +// to date). Used by actions/github-script from the `benchmark` workflow (PRs +// from this repository) and from the `benchmark comment` workflow (PRs from +// forks and dependabot). const fs = require('node:fs'); -const MARKER = ''; const MAX_BODY = 65536; /** @@ -11,29 +11,44 @@ const MAX_BODY = 65536; * @param {import('@octokit/rest').Octokit} options.github * @param {object} options.context * @param {object} options.core + * @param {string} options.marker HTML comment that identifies the comment + * @param {string} options.title heading of the failure message * @param {number} options.pr PR number * @param {string} options.sha head commit of the PR - * @param {string} options.runUrl URL of the benchmark run - * @param {boolean} options.success whether the benchmark run succeeded + * @param {string} options.runUrl URL of the workflow run + * @param {boolean} options.success whether the job that wrote the report succeeded * @param {string} [options.reportPath] Markdown report (required on success) + * @param {boolean} [options.create] false: only update an existing comment */ -module.exports = async ({ github, context, core, pr, sha, runUrl, success, reportPath }) => { +module.exports = async ({ + github, + context, + core, + marker, + title, + pr, + sha, + runUrl, + success, + reportPath, + create = true, +}) => { const { owner, repo } = context.repo; const short = sha.slice(0, 7); let report = success ? fs.readFileSync(reportPath, 'utf8') : [ - '## โฑ๏ธ Benchmark results', + `## ${title}`, '', '> [!CAUTION]', - `> The benchmark failed for \`${short}\`. See the [workflow run](${runUrl}).`, + `> The check failed for \`${short}\`. See the [workflow run](${runUrl}).`, ].join('\n'); const footer = `\n\n[Workflow run](${runUrl}) ยท updated for \`${short}\``; - const limit = MAX_BODY - MARKER.length - footer.length - 1; + const limit = MAX_BODY - marker.length - footer.length - 1; if (report.length > limit) report = report.slice(0, limit - 20) + '\n\nโ€ฆ(truncated)'; - const body = `${MARKER}\n${report}${footer}`; + const body = `${marker}\n${report}${footer}`; const comments = await github.paginate(github.rest.issues.listComments, { owner, @@ -42,14 +57,16 @@ module.exports = async ({ github, context, core, pr, sha, runUrl, success, repor per_page: 100, }); const existing = comments.find( - (c) => c.user?.login === 'github-actions[bot]' && c.body?.startsWith(MARKER), + (c) => c.user?.login === 'github-actions[bot]' && c.body?.startsWith(marker), ); if (existing) { await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); core.info(`Updated comment ${existing.html_url}`); - } else { + } else if (create) { const { data } = await github.rest.issues.createComment({ owner, repo, issue_number: pr, body }); core.info(`Created comment ${data.html_url}`); + } else { + core.info('Nothing to report and no comment to update, skipping.'); } }; diff --git a/.github/workflows/benchmark-comment.yml b/.github/workflows/benchmark-comment.yml index 3d8a8ed..16227fe 100644 --- a/.github/workflows/benchmark-comment.yml +++ b/.github/workflows/benchmark-comment.yml @@ -1,10 +1,11 @@ name: benchmark comment -# Posts the report of the `benchmark` workflow on PRs from forks and dependabot, -# which get a read-only token in `pull_request` (PRs from this repository are -# handled by the `comment` job of `benchmark`). It runs from the default -# branch with a write token, so it never checks out or runs code from the PR: -# the artifact is only text, and the PR is resolved from the run metadata. +# Posts the reports of the `benchmark` workflow (benchmarks and public API) on +# PRs from forks and dependabot, which get a read-only token in `pull_request` +# (PRs from this repository are handled by the `comment` and `api_comment` +# jobs of `benchmark`). It runs from the default branch with a write token, so +# it never checks out or runs code from the PR: the artifacts are only text +# and JSON data, and the PR is resolved from the run metadata. on: workflow_run: @@ -44,9 +45,22 @@ jobs: run-id: ${{ github.event.workflow_run.id }} github-token: ${{ github.token }} + # the api job can succeed when a benchmark fails: no artifact when it failed + - uses: actions/download-artifact@v8 + id: api + continue-on-error: true + with: + name: api + path: ${{ runner.temp }}/api + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - uses: actions/github-script@v9 + env: + API: ${{ steps.api.outcome == 'success' }} with: script: | + const fs = require('node:fs'); const run = context.payload.workflow_run; const { owner, repo } = context.repo; @@ -67,13 +81,27 @@ jobs: return; } - await require('./.github/scripts/benchmark-comment.cjs')({ - github, - context, - core, - pr, - sha: run.head_sha, - runUrl: run.html_url, + const comment = require('./.github/scripts/pr-comment.cjs'); + const common = { github, context, core, pr, sha: run.head_sha, runUrl: run.html_url }; + + await comment({ + ...common, + marker: '', + title: 'โฑ๏ธ Benchmark results', success: run.conclusion === 'success', reportPath: `${process.env.RUNNER_TEMP}/benchmark/report.md`, }); + + // only a boolean is read from the JSON of the PR run + if (process.env.API === 'true') { + const dir = `${process.env.RUNNER_TEMP}/api`; + const { changed } = JSON.parse(fs.readFileSync(`${dir}/api.json`, 'utf8')); + await comment({ + ...common, + marker: '', + title: '๐Ÿงฉ Public API changes', + success: true, + reportPath: `${dir}/report.md`, + create: changed === true, + }); + } diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 827595f..ed651bc 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,14 +1,19 @@ name: benchmark -# Runs the benchmarks of the PR against the latest release on npm and posts -# the report as a PR comment (one comment, kept up to date). PRs from forks and -# dependabot get a read-only token, so the `benchmark comment` workflow posts -# for them. +# Compares the PR against the latest release on npm and posts two PR comments +# (each one kept up to date): +# - the benchmarks of every runtime; +# - the changes in the public API (type declarations and package.json entry +# points). Posted only when the API changes, or to update an earlier one. +# PRs from forks and dependabot get a read-only token, so the +# `benchmark comment` workflow posts for them. on: pull_request: branches: - master + # every input of the build, so a PR that skips the workflow cannot change + # the API either paths: - "src/**" - "Cargo.toml" @@ -19,10 +24,12 @@ on: - "bin/wasm-opt.sh" - "bin/install-binaryen.sh" - "js/**" + - "tsconfig.json" - "tsdown.config.ts" - "bin/bench-*.mjs" + - "bin/api-diff.mjs" - "test/browser/**" - - ".github/scripts/benchmark-comment.cjs" + - ".github/scripts/pr-comment.cjs" - ".github/workflows/benchmark.yml" concurrency: @@ -82,8 +89,8 @@ jobs: curl -fsSL -o "$RUNNER_TEMP/rregex.tgz" "$(jq -r .dist.tarball <<< "$meta")" test "sha512-$(openssl dgst -sha512 -binary "$RUNNER_TEMP/rregex.tgz" | base64 -w0)" = "$(jq -r .dist.integrity <<< "$meta")" mkdir -p .latest - tar xzf "$RUNNER_TEMP/rregex.tgz" -C .latest --strip-components=1 package/lib - test -f .latest/lib/esm.mjs -a -f .latest/lib/rregex.wasm + tar xzf "$RUNNER_TEMP/rregex.tgz" -C .latest --strip-components=1 package/lib package/package.json + test -f .latest/lib/esm.mjs -a -f .latest/lib/rregex.wasm -a -f .latest/package.json echo "version=$(jq -r .version <<< "$meta")" >> "$GITHUB_OUTPUT" - name: installing @@ -114,6 +121,7 @@ jobs: path: | lib/ .latest/lib/ + .latest/package.json build.json include-hidden-files: true # .latest if-no-files-found: error @@ -177,6 +185,101 @@ jobs: if-no-files-found: error retention-days: 1 + # the public API of head vs the latest release, next to the benchmarks + api: + needs: build + + runs-on: ubuntu-24.04 + timeout-minutes: 5 + + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + LATEST_VERSION: ${{ needs.build.outputs.latest }} + + steps: + # api-diff.mjs only needs bin/, package.json, Node and git + - uses: actions/checkout@v7 + with: + sparse-checkout: bin + + - uses: actions/setup-node@v7 + with: + node-version: "26" + + - uses: actions/download-artifact@v8 + with: + name: bench-libs + + - name: comparing the public API + env: + REPORT_DIR: ${{ runner.temp }}/api + run: | + mkdir -p "$REPORT_DIR" + node bin/api-diff.mjs \ + --base .latest/lib \ + --head lib \ + --base-package .latest/package.json \ + --head-package package.json \ + --base-label "\`rregex@${LATEST_VERSION}\` (npm latest)" \ + --head-label "\`${HEAD_SHA::7}\`" \ + --markdown "$REPORT_DIR/report.md" \ + --json "$REPORT_DIR/api.json" + cat "$REPORT_DIR/report.md" >> "$GITHUB_STEP_SUMMARY" + + # the `benchmark comment` workflow downloads it by this name + - uses: actions/upload-artifact@v7 + with: + name: api + path: ${{ runner.temp }}/api + retention-days: 7 + + api_comment: + # PRs from this repository only, see `benchmark comment` for the others + if: >- + needs.api.result == 'success' && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login != 'dependabot[bot]' + + needs: api + + runs-on: ubuntu-24.04 + + permissions: + contents: read + pull-requests: write # to create and update the comment + + steps: + - uses: actions/checkout@v7 + with: + sparse-checkout: .github/scripts + + - uses: actions/download-artifact@v8 + with: + name: api + path: ${{ runner.temp }}/api + + # a new comment only when the API changes; an earlier one is updated + # either way, so it does not show changes that the PR no longer has + - uses: actions/github-script@v9 + with: + script: | + const fs = require('node:fs'); + const dir = `${process.env.RUNNER_TEMP}/api`; + const { changed } = JSON.parse(fs.readFileSync(`${dir}/api.json`, 'utf8')); + await require('./.github/scripts/pr-comment.cjs')({ + github, + context, + core, + marker: '', + title: '๐Ÿงฉ Public API changes', + pr: context.payload.pull_request.number, + sha: context.payload.pull_request.head.sha, + runUrl: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + success: true, + reportPath: `${dir}/report.md`, + create: changed === true, + }); + report: needs: [build, bench] @@ -265,10 +368,12 @@ jobs: SUCCESS: ${{ needs.report.result == 'success' }} with: script: | - await require('./.github/scripts/benchmark-comment.cjs')({ + await require('./.github/scripts/pr-comment.cjs')({ github, context, core, + marker: '', + title: 'โฑ๏ธ Benchmark results', pr: context.payload.pull_request.number, sha: context.payload.pull_request.head.sha, runUrl: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, diff --git a/bin/api-diff.mjs b/bin/api-diff.mjs new file mode 100755 index 0000000..5b05e06 --- /dev/null +++ b/bin/api-diff.mjs @@ -0,0 +1,187 @@ +#!/usr/bin/env node +// Compares the public API of two rregex builds: the type declarations in +// lib/ (*.d.ts, *.d.mts, *.d.cts) and the entry points of package.json +// (`exports`, `main`, `module`, `types`, `browser`). Writes a Markdown report +// with one diff for each change; files with the same change share one diff. +// +// Usage: +// node bin/api-diff.mjs --base --head \ +// [--base-package ] [--head-package ] \ +// [--base-label ] [--head-label ] \ +// [--markdown ] [--json ] +// +// The JSON file has `changed: true` when the API changed. Needs `git`. + +import { spawnSync } from 'node:child_process'; +import { mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { parseArgs } from 'node:util'; + +const { values: args } = parseArgs({ + options: { + base: { type: 'string' }, + head: { type: 'string' }, + 'base-package': { type: 'string' }, + 'head-package': { type: 'string' }, + 'base-label': { type: 'string', default: 'base' }, + 'head-label': { type: 'string', default: 'head' }, + markdown: { type: 'string' }, + json: { type: 'string' }, + }, +}); + +if (!args.base || !args.head) { + console.error('Usage: api-diff.mjs --base --head [options]'); + process.exit(2); +} + +// the comment limit of GitHub is 65536 chars, keep room for the footer +const MAX_MARKDOWN = 60000; +const ENTRY_FIELDS = ['main', 'module', 'types', 'browser', 'exports']; +const DTS = /\.d\.[cm]?ts$/; + +/** Unified diff of two files (null: the file does not exist), without the headers. */ +function diff(base, head) { + const { status, stdout, stderr } = spawnSync( + 'git', + ['diff', '--no-index', '--no-color', '--unified=3', '--', base ?? '/dev/null', head ?? '/dev/null'], + { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, + ); + // 0: same content, 1: different, anything else: git failed + if (status !== 0 && status !== 1) throw new Error(`git diff failed (${status}): ${stderr}`); + const lines = stdout.split('\n'); + const start = lines.findIndex((line) => line.startsWith('@@')); + return start === -1 ? '' : lines.slice(start).join('\n').trimEnd(); +} + +const listDts = async (dir) => (await readdir(dir)).filter((name) => DTS.test(name)); + +const dirs = { base: resolve(args.base), head: resolve(args.head) }; +const names = { + base: new Set(await listDts(dirs.base)), + head: new Set(await listDts(dirs.head)), +}; +const all = [...new Set([...names.base, ...names.head])].sort(); + +/** + * Changes, grouped by identical diff: { files, status, diff }. The line + * numbers of the hunks do not count, as the entry points have the same + * declarations at different lines. A removed file keeps no diff: its content + * is the base, not a change to review. + */ +const groups = new Map(); +const add = (file, status, body) => { + const key = `${status}\n${body.replace(/^@@ [^@]+ @@/gm, '@@')}`; + if (!groups.has(key)) groups.set(key, { files: [], status, diff: body }); + groups.get(key).files.push(file); +}; + +for (const name of all) { + const base = names.base.has(name) ? join(dirs.base, name) : null; + const head = names.head.has(name) ? join(dirs.head, name) : null; + if (!head) { + add(name, 'removed', ''); + continue; + } + const body = diff(base, head); + if (body) add(name, base ? 'modified' : 'added', body); +} + +// the entry points of package.json, as formatted JSON +if (args['base-package'] && args['head-package']) { + const tmp = await mkdtemp(join(tmpdir(), 'rregex-api-')); + try { + const entries = async (file) => { + const pkg = JSON.parse(await readFile(file, 'utf8')); + return JSON.stringify(Object.fromEntries(ENTRY_FIELDS.map((key) => [key, pkg[key]])), null, 2) + '\n'; + }; + const base = join(tmp, 'base.json'); + const head = join(tmp, 'head.json'); + await writeFile(base, await entries(args['base-package'])); + await writeFile(head, await entries(args['head-package'])); + const body = diff(base, head); + if (body) add('package.json', 'modified', body); + } finally { + await rm(tmp, { recursive: true, force: true }); + } +} + +const changes = [...groups.values()]; + +// ----- report ----- + +/** A code fence longer than any run of backticks in the text (JSDoc has ``` examples). */ +const fence = (text) => '`'.repeat(Math.max(3, ...[...text.matchAll(/`+/g)].map(([m]) => m.length + 1))); +const count = (body, sign) => body.split('\n').filter((line) => line.startsWith(sign)).length; +const files = (list) => list.map((file) => `\`${file}\``).join(', '); +const verb = { added: 'New file', removed: 'Removed file', modified: 'Changed' }; + +const lines = ['## ๐Ÿงฉ Public API changes', '']; +lines.push( + `Comparing the type declarations and the \`package.json\` entry points of **head** ${args['head-label']} ` + + `against **base** ${args['base-label']}.`, + '', +); + +if (!changes.length) { + lines.push('> [!NOTE]', '> No changes in the public API.'); +} else { + const total = changes.reduce((sum, change) => sum + change.files.length, 0); + lines.push( + '> [!IMPORTANT]', + `> The public API changes in **${total} file(s)**. In the diffs, lines that start with ` + + '`+` are added and lines that start with `-` are removed.', + '', + ); + + const truncated = []; + for (const change of changes) { + if (change.status === 'removed') { + lines.push(`### ${verb.removed}: ${files(change.files)}`, ''); + continue; + } + const added = count(change.diff, '+'); + const removed = count(change.diff, '-'); + const ticks = fence(change.diff); + const block = [ + `### ${verb[change.status]}: ${files(change.files)}`, + '', + `${added} line(s) added (+), ${removed} line(s) removed (-)` + + (change.files.length > 1 ? `; line numbers of \`${change.files[0]}\`` : '') + + '', + '', + `${ticks}diff`, + change.diff, + ticks, + '', + ]; + const size = block.reduce((sum, line) => sum + line.length + 1, 0); + if (lines.join('\n').length + size > MAX_MARKDOWN) { + truncated.push(...change.files); + continue; + } + lines.push(...block); + } + if (truncated.length) { + lines.push( + '> [!WARNING]', + `> The diffs of ${files(truncated)} are too long for this comment. ` + + 'The `api.json` file of the `api` artifact of the workflow run has all the diffs.', + '', + ); + } +} + +const markdown = lines.join('\n').trimEnd() + '\n'; +const json = { changed: changes.length > 0, changes }; + +if (args.markdown) await writeFile(args.markdown, markdown); +if (args.json) await writeFile(args.json, JSON.stringify(json, null, 2) + '\n'); +if (!args.markdown) process.stdout.write(markdown); + +if (process.env.GITHUB_ACTIONS && changes.length) { + for (const change of changes) { + console.log(`::notice title=Public API change::${verb[change.status]}: ${change.files.join(', ')}`); + } +}