diff --git a/.github/workflows/benchmark-node-scenario-batching.yml b/.github/workflows/benchmark-node-scenario-batching.yml new file mode 100644 index 000000000000..43111986d77f --- /dev/null +++ b/.github/workflows/benchmark-node-scenario-batching.yml @@ -0,0 +1,100 @@ +name: 'CI: Benchmark integration scenario batching' +on: + pull_request: + paths: + - '.github/workflows/benchmark-node-scenario-batching.yml' + - 'dev-packages/node-integration-tests/suites/tracing/google-genai/**' + - 'dev-packages/node-integration-tests/suites/tracing/pi-durable/**' + +permissions: + contents: read + actions: read + +jobs: + benchmark: + name: Profile ${{ matrix.suite }} (${{ matrix.label }}) + runs-on: ubuntu-24.04-arm + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + label: ['Node 20', 'Node 24'] + suite: ['tracing/google-genai', 'tracing/pi-durable'] + exclude: + - label: Node 20 + suite: tracing/pi-durable + include: + - label: Node 20 + node: '20.19' + - label: Node 24 + node: '24' + - label: Node 22 + node: '22' + suite: tracing/pi-durable + env: + BASELINE_REF: e221f40bf4ef5a6b4abe3e4315987bd6abd0718d + CHANGED_REF: ${{ github.event.pull_request.head.sha }} + CACHED_DEPENDENCY_PATHS: | + ${{ github.workspace }}/node_modules + ${{ github.workspace }}/packages/*/node_modules + ${{ github.workspace }}/dev-packages/*/node_modules + ~/.cache/mongodb-binaries/ + DISABLE_V8_COMPILE_CACHE: '1' + PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: 'false' + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 8 + - uses: actions/setup-node@v7 + with: + node-version: ${{ matrix.node }} + - uses: ./.github/actions/install-pnpm + - name: Compute dependency cache key + id: dependencies + run: node scripts/dependency-hash-key.js >> "$GITHUB_OUTPUT" + - uses: actions/cache/restore@v6.1.0 + id: cache + with: + path: ${{ env.CACHED_DEPENDENCY_PATHS }} + key: ${{ steps.dependencies.outputs.hash }}-${{ runner.arch }} + - if: steps.cache.outputs.cache-hit != 'true' + run: pnpm install --frozen-lockfile + - name: Restore common SDK build + uses: actions/download-artifact@v8 + with: + name: build-output + run-id: 43719995516 + github-token: ${{ github.token }} + - name: Compare original and batched suites + working-directory: dev-packages/node-integration-tests + env: + PROFILE_SUITE: ${{ matrix.suite }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/batching-results" + files=( + dev-packages/node-integration-tests/suites/tracing/google-genai + dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts + ) + printf '%s\n' "$PROFILE_SUITE" > "$RUNNER_TEMP/batching-results/suite.txt" + iteration=0 + for variant in warmup baseline changed changed baseline; do + iteration=$((iteration + 1)) + revision="$BASELINE_REF" + if [ "$variant" = changed ]; then revision="$CHANGED_REF"; fi + git -C "$GITHUB_WORKSPACE" restore --source="$revision" -- "${files[@]}" + measurement_started=$(date +%s%N) + pnpm run test \ + "suites/$PROFILE_SUITE/test.ts" \ + --maxWorkers=1 --reporter=verbose --reporter=json \ + --outputFile="$RUNNER_TEMP/batching-results/$iteration-$variant.json" + measurement_finished=$(date +%s%N) + printf '%s,%s,%s\n' "$iteration" "$variant" "$(((measurement_finished - measurement_started) / 1000000))" >> "$RUNNER_TEMP/batching-results/wall-times.csv" + done + - name: Upload suite timings + if: always() + uses: actions/upload-artifact@v7 + with: + name: batching-suite-timings-${{ strategy.job-index }}-node${{ matrix.node }} + path: ${{ runner.temp }}/batching-results/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 93faffc0ee3f..4f64eff72147 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,3 +1,4 @@ +# Benchmark baseline for batching integration scenarios. name: 'CI: Build & Test' on: push: @@ -153,6 +154,40 @@ jobs: $GITHUB_OUTPUT working-directory: dev-packages/e2e-tests + - name: Build tarballs and bundles + run: | + pnpm run clean:tarballs + pnpm exec nx run-many -t build:tarball build:bundle --parallel=4 + + - name: Upload tarball artifacts + uses: actions/upload-artifact@v7 + with: + name: build-tarball-output + path: ${{ env.TARBALL_ARTIFACT_GLOB }} + if-no-files-found: error + retention-days: 4 + compression-level: 6 + overwrite: true + + - name: Upload test-utils tarball artifact + uses: actions/upload-artifact@v7 + with: + name: build-test-utils-tarball-output + path: dev-packages/test-utils/*.tgz + if-no-files-found: error + retention-days: 4 + compression-level: 6 + overwrite: true + + - name: Upload bundle artifacts + uses: actions/upload-artifact@v7 + with: + name: build-bundle-output + path: ${{ env.BUNDLE_ARTIFACT_GLOB }} + retention-days: 4 + compression-level: 6 + overwrite: true + outputs: dependency_cache_key: ${{ steps.install_dependencies.outputs.cache_key }} changed_node_integration: @@ -230,43 +265,6 @@ jobs: compression-level: 6 overwrite: true - job_build_bundles: - name: Build bundles - needs: [job_get_metadata, job_build] - timeout-minutes: 10 - runs-on: ubuntu-24.04-arm - steps: - - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) - uses: actions/checkout@v7 - with: - ref: ${{ env.HEAD_COMMIT }} - - name: Set up Node - uses: ./.github/actions/setup-node - - name: Restore NX cache - uses: actions/cache/restore@v6.1.0 - with: - path: | - .nx/cache - .nx/workspace-data - key: ${{ env.NX_CACHE_KEY }}-${{ runner.arch }} - - - name: Restore caches - uses: ./.github/actions/restore-cache - with: - dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - - name: Build bundles - run: pnpm run build:bundle - - - name: Upload build artifacts - uses: actions/upload-artifact@v7 - with: - name: build-bundle-output - path: ${{ env.BUNDLE_ARTIFACT_GLOB }} - retention-days: 4 - compression-level: 6 - overwrite: true - job_check_branches: name: Check PR branches needs: job_get_metadata @@ -284,7 +282,7 @@ jobs: job_size_check: name: Size Check - needs: [job_get_metadata, job_build, job_build_bundles] + needs: [job_get_metadata, job_build] timeout-minutes: 15 runs-on: ubuntu-24.04-arm if: @@ -404,7 +402,7 @@ jobs: job_artifacts: name: Upload Artifacts - needs: [job_get_metadata, job_build, job_build_layer, job_build_bundles, job_build_tarballs] + needs: [job_get_metadata, job_build, job_build_layer] runs-on: ubuntu-24.04 # Build artifacts are only needed for releasing workflow. if: needs.job_get_metadata.outputs.is_release == 'true' @@ -592,7 +590,7 @@ jobs: name: Playwright ${{ matrix.bundle }}${{ matrix.project && matrix.project != 'chromium' && format(' {0}', matrix.project) || ''}}${{ matrix.shard && format(' ({0}/{1})', matrix.shard, matrix.shards) || ''}} Tests - needs: [job_get_metadata, job_build, job_build_bundles] + needs: [job_get_metadata, job_build] if: needs.job_build.outputs.changed_browser_integration == 'true' || github.event_name != 'pull_request' runs-on: ubuntu-24.04-large-js timeout-minutes: 25 @@ -610,8 +608,6 @@ jobs: - bundle_tracing - bundle_tracing_logs_metrics - bundle_tracing_replay - - bundle_tracing_replay_logs_metrics - - bundle_tracing_replay_feedback - bundle_tracing_replay_feedback_logs_metrics - bundle_tracing_replay_feedback_logs_metrics_min project: @@ -630,19 +626,51 @@ jobs: - bundle: esm project: chromium shard: 1 - shards: 4 + shards: 6 - bundle: esm project: chromium shard: 2 - shards: 4 + shards: 6 - bundle: esm project: chromium shard: 3 - shards: 4 + shards: 6 - bundle: esm project: chromium shard: 4 - shards: 4 + shards: 6 + - bundle: esm + project: chromium + shard: 5 + shards: 6 + - bundle: esm + project: chromium + shard: 6 + shards: 6 + - bundle: bundle_tracing_replay_feedback + project: chromium + shard: 1 + shards: 3 + - bundle: bundle_tracing_replay_feedback + project: chromium + shard: 2 + shards: 3 + - bundle: bundle_tracing_replay_feedback + project: chromium + shard: 3 + shards: 3 + - bundle: bundle_tracing_replay_logs_metrics + project: chromium + shard: 1 + shards: 3 + - bundle: bundle_tracing_replay_logs_metrics + project: chromium + shard: 2 + shards: 3 + - bundle: bundle_tracing_replay_logs_metrics + project: chromium + shard: 3 + shards: 3 steps: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) @@ -689,7 +717,7 @@ jobs: job_browser_loader_tests: name: PW ${{ matrix.bundle }} Tests - needs: [job_get_metadata, job_build, job_build_bundles] + needs: [job_get_metadata, job_build] if: needs.job_build.outputs.changed_browser_integration == 'true' || github.event_name != 'pull_request' runs-on: ubuntu-24.04 timeout-minutes: 15 @@ -748,7 +776,7 @@ jobs: # Runs the CDN bundles in the oldest Chromium we support. Chromium snapshots only exist for x64. job_browser_min_version_tests: name: Minimum Chromium Version Tests - needs: [job_get_metadata, job_build, job_build_bundles] + needs: [job_get_metadata, job_build] if: needs.job_build.outputs.changed_browser_integration == 'true' || github.event_name != 'pull_request' runs-on: ubuntu-24.04 timeout-minutes: 5 @@ -790,8 +818,8 @@ jobs: fail-fast: false matrix: node: [20.19, 22, 24, 26, '^27.0.0-0'] - shard: [1, 2, 3, 4] - shards: [4] + shard: [1, 2, 3, 4, 5, 6] + shards: [6] steps: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v7 @@ -941,7 +969,7 @@ jobs: run: pnpm run test job_bun_integration_tests: - name: Bun (${{ matrix.bun }}) Integration Tests (${{ matrix.shard }}/2) + name: Bun (${{ matrix.bun }}) Integration Tests (${{ matrix.shard }}/4) needs: [job_get_metadata, job_build] if: needs.job_build.outputs.changed_bun_integration == 'true' || github.event_name != 'pull_request' runs-on: ubuntu-24.04-arm @@ -951,7 +979,7 @@ jobs: matrix: # 1.3.14 is the oldest supported Bun version bun: ['1.3.14', '1.4'] - shard: [1, 2] + shard: [1, 2, 3, 4] steps: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v7 @@ -970,10 +998,10 @@ jobs: - name: Run integration tests working-directory: dev-packages/bun-integration-tests - run: pnpm run test --shard=${{ matrix.shard }}/2 + run: pnpm run test --shard=${{ matrix.shard }}/4 job_deno_integration_tests: - name: Deno Integration Tests (${{ matrix.shard }}/2) + name: Deno Integration Tests (${{ matrix.shard }}/4) needs: [job_get_metadata, job_build] if: needs.job_build.outputs.changed_deno_integration == 'true' || github.event_name != 'pull_request' runs-on: ubuntu-24.04-arm @@ -981,7 +1009,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3, 4] steps: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v7 @@ -1010,73 +1038,14 @@ jobs: - name: Run shared Node integration tests working-directory: dev-packages/deno-integration-tests # Tedious runs in the dedicated SQL Server job because its database container requires x86. - run: pnpm run test:node-suites --shard=${{ matrix.shard }}/2 --exclude '**/tracing/tedious/**' - - job_build_tarballs: - name: Build tarballs - # We want to run this if: - # - The build job was successful, not skipped - if: | - always() && - needs.job_build.result == 'success' - needs: [job_get_metadata, job_build] - runs-on: ubuntu-24.04-arm - timeout-minutes: 15 - steps: - - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) - uses: actions/checkout@v7 - with: - ref: ${{ env.HEAD_COMMIT }} - - name: Set up Node - uses: ./.github/actions/setup-node - - name: Restore NX cache - uses: actions/cache/restore@v6.1.0 - with: - path: | - .nx/cache - .nx/workspace-data - key: ${{ env.NX_CACHE_KEY }}-${{ runner.arch }} - - - name: Restore caches - uses: ./.github/actions/restore-cache - with: - dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - - name: Build tarballs - run: | - pnpm run clean:tarballs - pnpm run build:tarballs --parallel=4 - - - name: Upload tarball artifacts - uses: actions/upload-artifact@v7 - with: - name: build-tarball-output - path: ${{ env.TARBALL_ARTIFACT_GLOB }} - if-no-files-found: error - retention-days: 4 - compression-level: 6 - overwrite: true - - - name: Upload test-utils tarball artifact - uses: actions/upload-artifact@v7 - with: - name: build-test-utils-tarball-output - path: dev-packages/test-utils/*.tgz - if-no-files-found: error - retention-days: 4 - compression-level: 6 - overwrite: true + run: pnpm run test:node-suites --shard=${{ matrix.shard }}/4 --exclude '**/tracing/tedious/**' # GitHub Actions allows at most 256 jobs in one matrix, so the E2E tests are split into two jobs that share `env` # and `steps`. job_e2e_tests_1: name: E2E ${{ matrix.label || matrix.test-application }} Test - # We need to add the `always()` check here because the previous step has this as well :( - # See: https://github.com/actions/runner/issues/2205 - if: - always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix-1 != - '{"include":[]}' - needs: [job_get_metadata, job_build, job_build_tarballs] + if: always() && needs.job_build.result == 'success' && needs.job_build.outputs.e2e-matrix-1 != '{"include":[]}' + needs: [job_get_metadata, job_build] runs-on: ubuntu-24.04 timeout-minutes: 15 # We just use a dummy DSN here, only send to the tunnel anyhow @@ -1090,6 +1059,7 @@ jobs: E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' # The `runtime` of a runtime variant, see dev-packages/e2e-tests/README.md RUNTIME: ${{ matrix.runtime }} + SENTRY_E2E_TEST_SHARD: ${{ matrix.test-shard }} # Without the root node_modules, `pnpm run` would first install the whole workspace. pnpm_config_verify_deps_before_run: 'false' strategy: @@ -1146,10 +1116,8 @@ jobs: job_e2e_tests_2: name: E2E ${{ matrix.label || matrix.test-application }} Test - if: - always() && needs.job_build_tarballs.result == 'success' && needs.job_build.outputs.e2e-matrix-2 != - '{"include":[]}' - needs: [job_get_metadata, job_build, job_build_tarballs] + if: always() && needs.job_build.result == 'success' && needs.job_build.outputs.e2e-matrix-2 != '{"include":[]}' + needs: [job_get_metadata, job_build] runs-on: ubuntu-24.04 timeout-minutes: 15 env: *e2e-env @@ -1160,9 +1128,9 @@ jobs: job_e2e_aws_layer_tests: name: E2E ${{ matrix.label || matrix.test-application }} Test - needs: [job_get_metadata, job_build, job_build_layer, job_build_tarballs] + needs: [job_get_metadata, job_build, job_build_layer] if: >- - always() && needs.job_build_tarballs.result == 'success' && needs.job_build_layer.result == 'success' && + always() && needs.job_build.result == 'success' && needs.job_build_layer.result == 'success' && needs.job_build.outputs.e2e-matrix-aws-layer != '{"include":[]}' runs-on: ubuntu-24.04 timeout-minutes: 15 @@ -1178,13 +1146,11 @@ jobs: job_optional_e2e_tests: name: E2E Tests (optional) # We only run E2E tests for non-fork PRs because the E2E tests require secrets to work and they can't be accessed from forks - # We need to add the `always()` check here because the previous step has this as well :( - # See: https://github.com/actions/runner/issues/2205 if: - always() && needs.job_get_metadata.outputs.is_release != 'true' && needs.job_build_tarballs.result == 'success' && + always() && needs.job_get_metadata.outputs.is_release != 'true' && needs.job_build.result == 'success' && needs.job_build.outputs.e2e-matrix-optional != '{"include":[]}' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]' - needs: [job_get_metadata, job_build, job_build_tarballs] + needs: [job_get_metadata, job_build] # This job only waits for optional-e2e.yml, which can take most of an hour. runs-on: ubuntu-24.04 timeout-minutes: 60 @@ -1215,9 +1181,7 @@ jobs: needs: [ job_build, - job_build_bundles, job_build_layer, - job_build_tarballs, job_browser_unit_tests, job_bun_unit_tests, job_deno_unit_tests, diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 8e29e056370e..f8ef50f4da66 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -85,6 +85,7 @@ jobs: E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' # The `runtime` of a runtime variant, see dev-packages/e2e-tests/README.md RUNTIME: ${{ matrix.runtime }} + SENTRY_E2E_TEST_SHARD: ${{ matrix.test-shard }} # Without the root node_modules, `pnpm run` would first install the whole workspace. pnpm_config_verify_deps_before_run: 'false' strategy: diff --git a/.github/workflows/optional-e2e.yml b/.github/workflows/optional-e2e.yml index eea799a5a5f9..48c469acacb1 100644 --- a/.github/workflows/optional-e2e.yml +++ b/.github/workflows/optional-e2e.yml @@ -63,6 +63,7 @@ jobs: E2E_PR_NUMBER: ${{ inputs.pr_number }} # The `runtime` of a runtime variant, see dev-packages/e2e-tests/README.md RUNTIME: ${{ matrix.runtime }} + SENTRY_E2E_TEST_SHARD: ${{ matrix.test-shard }} # Without the root node_modules, `pnpm run` would first install the whole workspace. pnpm_config_verify_deps_before_run: 'false' strategy: diff --git a/dev-packages/e2e-tests/README.md b/dev-packages/e2e-tests/README.md index 52e30cf48569..3ba573d25039 100644 --- a/dev-packages/e2e-tests/README.md +++ b/dev-packages/e2e-tests/README.md @@ -299,6 +299,10 @@ of a test application. set its `assert-command` to run production tests and add a development variant to run both environments in parallel. The local `test:assert` script can still run both; select the base variant locally with `--variant