Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
88e0135
test(ci): Trigger full baseline for E2E sharding benchmark
nicohrubec Nov 14, 2026
af16632
test(ci): Shard slow Nuxt and Next.js E2E variants
nicohrubec Nov 14, 2026
e16aac8
test(ci): Shard the Nuxt Vite-server E2E variant
nicohrubec Nov 14, 2026
73c55df
test(ci): Keep sharding for the variants with larger gains
nicohrubec Nov 14, 2026
09594f9
chore(ci): Pack tarballs in the build job when needed
nicohrubec Nov 14, 2026
b2a3cda
chore(ci): Always pack tarballs during Build
nicohrubec Nov 14, 2026
e8c8e00
test(ci): Benchmark more integration test shards
nicohrubec Nov 14, 2026
034319b
test(ci): Keep WebKit integration tests on two shards
nicohrubec Nov 14, 2026
2e35c86
test(ci): Benchmark packing and bundle builds on one runner
nicohrubec Nov 14, 2026
e6fb59b
test(ci): Capture baseline for integration scenario batching
nicohrubec Nov 15, 2026
f3d366b
test(node): Batch compatible integration scenarios
nicohrubec Nov 15, 2026
b37402f
test(ci): Profile batched suites and distinguish captured errors
nicohrubec Nov 15, 2026
3a0dbd4
test(node): Batch request body checks on the slowest shard
nicohrubec Nov 15, 2026
3d0d6dd
test(node): Reuse the server for trace-header checks
nicohrubec Nov 15, 2026
3ffeb35
test(node): Share identical Vercel AI scenario output
nicohrubec Nov 15, 2026
71826a5
test(node): batch compatible AI integration scenarios
nicohrubec Nov 15, 2026
8693f97
test(node): match scenario roots when collecting batched spans
nicohrubec Nov 15, 2026
62d7a1d
test(node): flush once after batched scenarios
nicohrubec Nov 15, 2026
e13aa60
test(node): retain only beneficial AI scenario groups
nicohrubec Nov 15, 2026
f17a091
test(node): preserve strict event handling in batched scenarios
nicohrubec Nov 15, 2026
b6b8ccb
test(node): reuse Hono server for non-error request scenarios
nicohrubec Nov 15, 2026
10dc39f
test(node): batch filesystem instrumentation requests
nicohrubec Nov 15, 2026
7b9c357
test(node): share Express server for happy-path tracing cases
nicohrubec Nov 15, 2026
b11e003
test(node): batch compatible framework request scenarios
nicohrubec Nov 15, 2026
dbfb678
test(node): expand batching across compatible AI scenarios
nicohrubec Nov 15, 2026
e598be5
test(node): batch compatible scenarios across AI providers
nicohrubec Nov 15, 2026
bd05f5c
test(node): share GenAI and pi-durable scenario executions
nicohrubec Nov 15, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 100 additions & 0 deletions .github/workflows/benchmark-node-scenario-batching.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
name: 'CI: Benchmark integration scenario batching'
on:
pull_request:
paths:
- '.github/workflows/benchmark-node-scenario-batching.yml'
- 'dev-packages/node-integration-tests/suites/tracing/google-genai/**'
- 'dev-packages/node-integration-tests/suites/tracing/pi-durable/**'

permissions:
contents: read
actions: read

jobs:
benchmark:
name: Profile ${{ matrix.suite }} (${{ matrix.label }})
runs-on: ubuntu-24.04-arm
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
label: ['Node 20', 'Node 24']
suite: ['tracing/google-genai', 'tracing/pi-durable']
exclude:
- label: Node 20
suite: tracing/pi-durable
include:
- label: Node 20
node: '20.19'
- label: Node 24
node: '24'
- label: Node 22
node: '22'
suite: tracing/pi-durable
env:
BASELINE_REF: e221f40bf4ef5a6b4abe3e4315987bd6abd0718d
CHANGED_REF: ${{ github.event.pull_request.head.sha }}
CACHED_DEPENDENCY_PATHS: |
${{ github.workspace }}/node_modules
${{ github.workspace }}/packages/*/node_modules
${{ github.workspace }}/dev-packages/*/node_modules
~/.cache/mongodb-binaries/
DISABLE_V8_COMPILE_CACHE: '1'
PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: 'false'
steps:
- uses: actions/checkout@v7
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 8
- uses: actions/setup-node@v7
with:
node-version: ${{ matrix.node }}
- uses: ./.github/actions/install-pnpm
- name: Compute dependency cache key
id: dependencies
run: node scripts/dependency-hash-key.js >> "$GITHUB_OUTPUT"
- uses: actions/cache/restore@v6.1.0
id: cache
with:
path: ${{ env.CACHED_DEPENDENCY_PATHS }}
key: ${{ steps.dependencies.outputs.hash }}-${{ runner.arch }}
- if: steps.cache.outputs.cache-hit != 'true'
run: pnpm install --frozen-lockfile
- name: Restore common SDK build
uses: actions/download-artifact@v8
with:
name: build-output
run-id: 43719995516
github-token: ${{ github.token }}
- name: Compare original and batched suites
working-directory: dev-packages/node-integration-tests
env:
PROFILE_SUITE: ${{ matrix.suite }}
run: |
set -euo pipefail
mkdir -p "$RUNNER_TEMP/batching-results"
files=(
dev-packages/node-integration-tests/suites/tracing/google-genai
dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts
)
printf '%s\n' "$PROFILE_SUITE" > "$RUNNER_TEMP/batching-results/suite.txt"
iteration=0
for variant in warmup baseline changed changed baseline; do
iteration=$((iteration + 1))
revision="$BASELINE_REF"
if [ "$variant" = changed ]; then revision="$CHANGED_REF"; fi
git -C "$GITHUB_WORKSPACE" restore --source="$revision" -- "${files[@]}"
measurement_started=$(date +%s%N)
pnpm run test \
"suites/$PROFILE_SUITE/test.ts" \
--maxWorkers=1 --reporter=verbose --reporter=json \
--outputFile="$RUNNER_TEMP/batching-results/$iteration-$variant.json"
measurement_finished=$(date +%s%N)
printf '%s,%s,%s\n' "$iteration" "$variant" "$(((measurement_finished - measurement_started) / 1000000))" >> "$RUNNER_TEMP/batching-results/wall-times.csv"
done
- name: Upload suite timings
if: always()
uses: actions/upload-artifact@v7
with:
name: batching-suite-timings-${{ strategy.job-index }}-node${{ matrix.node }}
path: ${{ runner.temp }}/batching-results/*
Loading
Loading