Skip to content

test(ci): Benchmark batching integration scenarios #18

test(ci): Benchmark batching integration scenarios

test(ci): Benchmark batching integration scenarios #18

name: 'CI: Benchmark integration scenario batching'
on:
pull_request:
paths:
- '.github/workflows/benchmark-node-scenario-batching.yml'
- 'dev-packages/node-integration-tests/suites/tracing/google-genai/**'
- 'dev-packages/node-integration-tests/suites/tracing/pi-durable/**'
permissions:
contents: read
actions: read
jobs:
benchmark:
name: Profile ${{ matrix.suite }} (${{ matrix.label }})
runs-on: ubuntu-24.04-arm
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
label: ['Node 20', 'Node 24']
suite: ['tracing/google-genai', 'tracing/pi-durable']
exclude:
- label: Node 20
suite: tracing/pi-durable
include:
- label: Node 20
node: '20.19'
- label: Node 24
node: '24'
- label: Node 22
node: '22'
suite: tracing/pi-durable
env:
BASELINE_REF: e221f40bf4ef5a6b4abe3e4315987bd6abd0718d
CHANGED_REF: ${{ github.event.pull_request.head.sha }}
CACHED_DEPENDENCY_PATHS: |
${{ github.workspace }}/node_modules
${{ github.workspace }}/packages/*/node_modules
${{ github.workspace }}/dev-packages/*/node_modules
~/.cache/mongodb-binaries/
DISABLE_V8_COMPILE_CACHE: '1'
PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: 'false'
steps:
- uses: actions/checkout@v7
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 8
- uses: actions/setup-node@v7
with:
node-version: ${{ matrix.node }}
- uses: ./.github/actions/install-pnpm
- name: Compute dependency cache key
id: dependencies
run: node scripts/dependency-hash-key.js >> "$GITHUB_OUTPUT"
- uses: actions/cache/restore@v6.1.0
id: cache
with:
path: ${{ env.CACHED_DEPENDENCY_PATHS }}
key: ${{ steps.dependencies.outputs.hash }}-${{ runner.arch }}
- if: steps.cache.outputs.cache-hit != 'true'
run: pnpm install --frozen-lockfile
- name: Restore common SDK build
uses: actions/download-artifact@v8
with:
name: build-output
run-id: 43719995516
github-token: ${{ github.token }}
- name: Compare original and batched suites
working-directory: dev-packages/node-integration-tests
env:
PROFILE_SUITE: ${{ matrix.suite }}
run: |
set -euo pipefail
mkdir -p "$RUNNER_TEMP/batching-results"
files=(
dev-packages/node-integration-tests/suites/tracing/google-genai
dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts
)
printf '%s\n' "$PROFILE_SUITE" > "$RUNNER_TEMP/batching-results/suite.txt"
iteration=0
for variant in warmup baseline changed changed baseline; do
iteration=$((iteration + 1))
revision="$BASELINE_REF"
if [ "$variant" = changed ]; then revision="$CHANGED_REF"; fi
git -C "$GITHUB_WORKSPACE" restore --source="$revision" -- "${files[@]}"
measurement_started=$(date +%s%N)
pnpm run test \
"suites/$PROFILE_SUITE/test.ts" \
--maxWorkers=1 --reporter=verbose --reporter=json \
--outputFile="$RUNNER_TEMP/batching-results/$iteration-$variant.json"
measurement_finished=$(date +%s%N)
printf '%s,%s,%s\n' "$iteration" "$variant" "$(((measurement_finished - measurement_started) / 1000000))" >> "$RUNNER_TEMP/batching-results/wall-times.csv"
done
- name: Upload suite timings
if: always()
uses: actions/upload-artifact@v7
with:
name: batching-suite-timings-${{ strategy.job-index }}-node${{ matrix.node }}
path: ${{ runner.temp }}/batching-results/*