ci.yml

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

jobs:
  test:
    name: Test (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
      fail-fast: false

    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v2
        with:
          version: "latest"

      - name: Set up Python ${{ matrix.python-version }}
        run: uv python install ${{ matrix.python-version }}

      - name: Install dependencies
        run: uv sync --locked --all-extras --python ${{ matrix.python-version }}

      - name: Confirm Python version
        run: |
          uv run python --version
          uv run python -c "import sys; print(f'Running Python {sys.version}'); assert sys.version.startswith('${{ matrix.python-version }}')"

      - name: Check plugin contracts
        run: uv run python .github/scripts/plugin_contracts.py

      - name: List registered plugins
        run: uv run harness list-plugins

      - name: Run pytest with coverage
        run: uv run pytest --cov=harness --cov-report=xml

      - name: Validate benchmarks
        run: |
          uv run harness validate examples/basic/benchmark.yaml
          uv run harness validate benchmarks/examples/customer_support.yaml
          uv run harness validate benchmarks/examples/code_generation.yaml

      - name: Run sample evaluation
        run: uv run harness evaluate examples/basic/benchmark.yaml