eval-regression.yml

# Eval Regression CI
# Runs buffbench on every PR against a reduced task set, compares to main branch
# baseline, and posts a comment showing regressions/improvements.

name: Eval Regression

on:
  pull_request:
    branches: ['master']
  workflow_dispatch:

concurrency:
  group: eval-regression-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # Gate: model evals cost real money and an hour of wall time — only run
  # them when the PR actually touches the agent loop, prompts, tools, or
  # eval machinery. Docs/UI/web-only PRs skip straight past.
  changes:
    runs-on: ubuntu-latest
    outputs:
      should-run: ${{ steps.filter.outputs.should-run }}
    steps:
      - name: Check changed paths
        id: filter
        uses: dorny/paths-filter@v3
        with:
          filters: |
            should-run:
              - 'packages/agent-runtime/**'
              - 'common/src/util/**'
              - 'common/src/prompts/**'
              - 'common/src/agents/**'
              - 'common/src/tools/**'
              - 'common/src/context/**'
              - 'evals/**'
              - 'sdk/src/run.ts'
              - 'sdk/src/tools/**'
              - 'agents/**'

  eval-regression:
    needs: changes
    if: ${{ needs.changes.outputs.should-run == 'true' }}
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout PR
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: '1.3.5'

      - name: Cache dependencies
        uses: actions/cache@v4
        with:
          path: |
            node_modules
            */node_modules
            packages/*/node_modules
          key: ${{ runner.os }}-deps-${{ hashFiles('**/bun.lock*') }}
          restore-keys: |
            ${{ runner.os }}-deps-

      - name: Install dependencies
        run: bun install --frozen-lockfile

      - name: Set environment variables
        env:
          SECRETS_CONTEXT: ${{ toJSON(secrets) }}
        run: |
          VAR_NAMES=$(bun scripts/generate-ci-env.ts)
          echo "$SECRETS_CONTEXT" | jq -r --argjson vars "$VAR_NAMES" '
            to_entries | .[] | select(.key as $k | $vars | index($k)) | .key + "=" + .value
          ' >> $GITHUB_ENV
          echo "LEVELCODE_GITHUB_ACTIONS=true" >> $GITHUB_ENV
          echo "LEVELCODE_GITHUB_TOKEN=${{ secrets.LEVELCODE_GITHUB_TOKEN }}" >> $GITHUB_ENV

      - name: Build SDK
        run: cd sdk && bun run build

      - name: Save baseline (main branch)
        id: baseline
        run: |
          git stash
          git checkout master
          mkdir -p evals/buffbench/logs-baseline
          cd evals
          bun run buffbench/main.ts --eval-file buffbench/eval-plane.json --agents base2 --concurrency 2 2>&1 | tee baseline-run.log || true
          cp -r buffbench/logs buffbench/logs-baseline/ 2>/dev/null || true
          git checkout ${{ github.head_ref || github.ref_name }}
          git stash pop 2>/dev/null || true

      - name: Run PR evals
        id: pr-run
        run: |
          mkdir -p evals/buffbench/logs-pr
          cd evals
          bun run buffbench/main.ts --eval-file buffbench/eval-plane.json --agents base2 --concurrency 2 2>&1 | tee pr-run.log || true
          cp -r buffbench/logs buffbench/logs-pr/ 2>/dev/null || true

      - name: Compare results
        id: compare
        run: |
          cd evals
          BASELINE=$(find buffbench/logs-baseline/logs -name "FINAL_RESULTS.json" -type f 2>/dev/null | head -1)
          CURRENT=$(find buffbench/logs-pr/logs -name "FINAL_RESULTS.json" -type f 2>/dev/null | head -1)
          if [ -z "$BASELINE" ] || [ -z "$CURRENT" ]; then
            echo "### ⚠️ Eval Run Incomplete" > $GITHUB_OUTPUT
            echo "Could not locate FINAL_RESULTS.json for both runs." >> $GITHUB_OUTPUT
            echo "" >> $GITHUB_OUTPUT
            echo "Baseline found: ${BASELINE:-none}" >> $GITHUB_OUTPUT
            echo "Current found: ${CURRENT:-none}" >> $GITHUB_OUTPUT
            echo "has_report=false" >> $GITHUB_OUTPUT
          else
            REPORT=$(bun run ci/eval-compare.ts "$BASELINE" "$CURRENT" --markdown 2>&1 || true)
            echo "report<<EOF" >> $GITHUB_OUTPUT
            echo "$REPORT" >> $GITHUB_OUTPUT
            echo "EOF" >> $GITHUB_OUTPUT
            echo "has_report=true" >> $GITHUB_OUTPUT
          fi

      - name: Find existing comment
        id: find-comment
        uses: peter-evans/find-comment@v3
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: 'Eval Regression Report'

      - name: Post or update PR comment
        uses: peter-evans/create-or-update-comment@v4
        with:
          comment-id: ${{ steps.find-comment.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            ${{ steps.compare.outputs.report }}

            ---
            _Auto-generated by eval-regression workflow for commit ${{ github.event.pull_request.head.sha }} (vs master baseline)_
          edit-mode: replace