fix: enforce trustworthy quality snapshots #286
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Tests | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - release* | |
| - release/* | |
| - release-* | |
| workflow_dispatch: | |
| permissions: | |
| actions: read | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| performance: | |
| name: E2E Performance Tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: windows-latest | |
| target: x86_64-pc-windows-msvc | |
| platform: Windows | |
| comment_header: perf-windows | |
| - os: ubuntu-latest | |
| target: x86_64-unknown-linux-musl | |
| platform: Linux | |
| comment_header: perf-linux | |
| - os: macos-latest | |
| target: x86_64-apple-darwin | |
| platform: macOS | |
| comment_header: perf-macos | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Post In-Progress Comment | |
| if: github.event_name == 'pull_request' | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| header: ${{ matrix.comment_header }} | |
| message: | | |
| ## Performance Report (${{ matrix.platform }}) :hourglass_flowing_sand: | |
| Running performance tests against baseline `${{ github.event.pull_request.base.sha }}`. | |
| - name: Set Python to PATH | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Validate Snapshot Comparator | |
| run: python -m unittest discover -s scripts/tests -p 'test_*.py' -v | |
| shell: bash | |
| - name: Add Conda to PATH (Windows) | |
| if: startsWith(matrix.os, 'windows') | |
| run: | | |
| $path = $env:PATH + ";" + $env:CONDA + "\condabin" | |
| echo "PATH=$path" >> $env:GITHUB_ENV | |
| - name: Add Conda to PATH (Ubuntu) | |
| if: startsWith(matrix.os, 'ubuntu') | |
| run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Install Conda + add to PATH (macOS) | |
| if: startsWith(matrix.os, 'macos') | |
| run: | | |
| curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh | |
| bash ~/miniconda.sh -b -p ~/miniconda | |
| echo "PATH=$PATH:$HOME/miniconda/bin" >> $GITHUB_ENV | |
| echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Create test Conda environment | |
| run: conda create -n perf-test-env python=3.12 -y | |
| - name: Create test venv | |
| run: python -m venv .venv | |
| shell: bash | |
| - name: Rust Tool Chain setup | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| targets: ${{ matrix.target }} | |
| - name: Cargo Fetch | |
| run: cargo fetch | |
| shell: bash | |
| - name: Build Release | |
| run: cargo build --release --target ${{ matrix.target }} | |
| shell: bash | |
| - name: Run Performance Tests | |
| id: benchmark | |
| run: | | |
| set -o pipefail | |
| cargo test --release --features ci-perf --target ${{ matrix.target }} --test e2e_performance test_performance_summary -- --nocapture 2>&1 | tee perf-output.txt | |
| env: | |
| RUST_BACKTRACE: 1 | |
| RUST_LOG: warn | |
| shell: bash | |
| - name: Extract Performance Metrics | |
| if: steps.benchmark.outcome == 'success' | |
| run: | | |
| if ! grep -q "JSON metrics:" perf-output.txt; then | |
| echo "Performance test produced no JSON metrics" >&2 | |
| exit 1 | |
| fi | |
| sed -n '/JSON metrics:/,/^}/p' perf-output.txt | tail -n +2 > metrics.json | |
| python -m json.tool metrics.json > /dev/null | |
| cat metrics.json | |
| shell: bash | |
| - name: Upload PR Performance Results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: perf-pr-${{ matrix.os }} | |
| path: metrics.json | |
| if-no-files-found: ignore | |
| - name: Download Exact PR Base Performance | |
| if: always() && github.event_name == 'pull_request' | |
| uses: dawidd6/action-download-artifact@v6 | |
| with: | |
| workflow: perf-baseline.yml | |
| commit: ${{ github.event.pull_request.base.sha }} | |
| workflow_conclusion: success | |
| name: perf-baseline-${{ matrix.os }} | |
| path: baseline-perf | |
| check_artifacts: true | |
| search_artifacts: true | |
| - name: Download Main Performance for Manual Run | |
| if: always() && github.event_name == 'workflow_dispatch' | |
| uses: dawidd6/action-download-artifact@v6 | |
| with: | |
| workflow: perf-baseline.yml | |
| branch: main | |
| workflow_conclusion: success | |
| name: perf-baseline-${{ matrix.os }} | |
| path: baseline-perf | |
| check_artifacts: true | |
| search_artifacts: true | |
| - name: Compare Performance Snapshot | |
| if: always() | |
| run: >- | |
| python scripts/quality_snapshot.py performance | |
| --current metrics.json | |
| --baseline baseline-perf/metrics.json | |
| --platform "${{ matrix.platform }}" | |
| --report performance-report.md | |
| --summary "$GITHUB_STEP_SUMMARY" | |
| shell: bash | |
| - name: Post Performance Comment | |
| if: always() && github.event_name == 'pull_request' | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| header: ${{ matrix.comment_header }} | |
| path: performance-report.md |