Repository navigation
test(ci): Benchmark batching integration scenarios #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'CI: Benchmark integration scenario batching' | |
| on: | |
| pull_request: | |
| paths: | |
| - '.github/workflows/benchmark-node-scenario-batching.yml' | |
| - 'dev-packages/node-integration-tests/suites/tracing/google-genai/**' | |
| - 'dev-packages/node-integration-tests/suites/tracing/pi-durable/**' | |
| permissions: | |
| contents: read | |
| actions: read | |
| jobs: | |
| benchmark: | |
| name: Profile ${{ matrix.suite }} (${{ matrix.label }}) | |
| runs-on: ubuntu-24.04-arm | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| label: ['Node 20', 'Node 24'] | |
| suite: ['tracing/google-genai', 'tracing/pi-durable'] | |
| exclude: | |
| - label: Node 20 | |
| suite: tracing/pi-durable | |
| include: | |
| - label: Node 20 | |
| node: '20.19' | |
| - label: Node 24 | |
| node: '24' | |
| - label: Node 22 | |
| node: '22' | |
| suite: tracing/pi-durable | |
| env: | |
| BASELINE_REF: e221f40bf4ef5a6b4abe3e4315987bd6abd0718d | |
| CHANGED_REF: ${{ github.event.pull_request.head.sha }} | |
| CACHED_DEPENDENCY_PATHS: | | |
| ${{ github.workspace }}/node_modules | |
| ${{ github.workspace }}/packages/*/node_modules | |
| ${{ github.workspace }}/dev-packages/*/node_modules | |
| ~/.cache/mongodb-binaries/ | |
| DISABLE_V8_COMPILE_CACHE: '1' | |
| PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: 'false' | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 8 | |
| - uses: actions/setup-node@v7 | |
| with: | |
| node-version: ${{ matrix.node }} | |
| - uses: ./.github/actions/install-pnpm | |
| - name: Compute dependency cache key | |
| id: dependencies | |
| run: node scripts/dependency-hash-key.js >> "$GITHUB_OUTPUT" | |
| - uses: actions/cache/restore@v6.1.0 | |
| id: cache | |
| with: | |
| path: ${{ env.CACHED_DEPENDENCY_PATHS }} | |
| key: ${{ steps.dependencies.outputs.hash }}-${{ runner.arch }} | |
| - if: steps.cache.outputs.cache-hit != 'true' | |
| run: pnpm install --frozen-lockfile | |
| - name: Restore common SDK build | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: build-output | |
| run-id: 43719995516 | |
| github-token: ${{ github.token }} | |
| - name: Compare original and batched suites | |
| working-directory: dev-packages/node-integration-tests | |
| env: | |
| PROFILE_SUITE: ${{ matrix.suite }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p "$RUNNER_TEMP/batching-results" | |
| files=( | |
| dev-packages/node-integration-tests/suites/tracing/google-genai | |
| dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts | |
| ) | |
| printf '%s\n' "$PROFILE_SUITE" > "$RUNNER_TEMP/batching-results/suite.txt" | |
| iteration=0 | |
| for variant in warmup baseline changed changed baseline; do | |
| iteration=$((iteration + 1)) | |
| revision="$BASELINE_REF" | |
| if [ "$variant" = changed ]; then revision="$CHANGED_REF"; fi | |
| git -C "$GITHUB_WORKSPACE" restore --source="$revision" -- "${files[@]}" | |
| measurement_started=$(date +%s%N) | |
| pnpm run test \ | |
| "suites/$PROFILE_SUITE/test.ts" \ | |
| --maxWorkers=1 --reporter=verbose --reporter=json \ | |
| --outputFile="$RUNNER_TEMP/batching-results/$iteration-$variant.json" | |
| measurement_finished=$(date +%s%N) | |
| printf '%s,%s,%s\n' "$iteration" "$variant" "$(((measurement_finished - measurement_started) / 1000000))" >> "$RUNNER_TEMP/batching-results/wall-times.csv" | |
| done | |
| - name: Upload suite timings | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: batching-suite-timings-${{ strategy.job-index }}-node${{ matrix.node }} | |
| path: ${{ runner.temp }}/batching-results/* |