reference/.github/workflows/ci.yml at main · agentic-datasets/reference · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
name: conformance

on:
  push:
  pull_request:
  workflow_dispatch:

jobs:
  harness-standalone:
    # The portability claim rests on the conformance distribution needing no
    # implementation at all. This job installs *only* that package -- not the
    # reference implementation, not this repository -- and runs the suite.
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: pip install ./packages/agentic-dataset-conformance
      - name: the reference implementation is not installed
        run: |
          python - <<'PY'
          import importlib.util as u
          # find_spec on a dotted name imports the parent package first, so
          # find_spec("google.adk") raises ModuleNotFoundError when `google`
          # itself is absent -- which is the very state this job asserts. The
          # exception and a None spec mean the same thing here: not installed.
          for m in ("agentic_dataset", "langgraph", "llama_index", "google.adk", "mcp"):
              try:
                  spec = u.find_spec(m)
              except ModuleNotFoundError:
                  spec = None
              assert spec is None, f"{m} should not be installed in this job"
          PY
      - name: the suite runs against its own subject, from an unrelated directory
        run: cd /tmp && agentic-dataset-conformance run --matrix
      - name: the vectors can be exported for use elsewhere
        run: cd /tmp && agentic-dataset-conformance vectors --export ./exported && ls ./exported/vectors | wc -l

  packages-build:
    # Both distributions must build and install standalone, wheel and sdist.
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - package: agentic-dataset-conformance
            module: agentic_dataset_conformance
          - package: authorized-recall
            module: authorized_recall
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: pip install build
      - run: python -m build --outdir dist ./packages/${{ matrix.package }}
      - name: wheel installs standalone and imports
        run: |
          python -m venv /tmp/w
          /tmp/w/bin/pip install dist/*.whl
          cd /tmp && /tmp/w/bin/python -c "import ${{ matrix.module }} as m; print(m.__name__, m.__version__)"
      - name: sdist installs standalone and imports
        run: |
          python -m venv /tmp/s
          /tmp/s/bin/pip install dist/*.tar.gz
          cd /tmp && /tmp/s/bin/python -c "import ${{ matrix.module }} as m; print(m.__name__, m.__version__)"

  all-subjects:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - run: pip install -e ./packages/authorized-recall -e ./packages/agentic-dataset-conformance -e ".[all]"
      - name: portable suite, every subject, plus the mutation matrix
        run: agentic-dataset-conformance run --subject conformance.subjects:subjects --matrix
      - name: white-box suite, 8 configurations
        run: python -m agentic_dataset.reference_suite
      - run: pytest -q
      - name: evaluation and the M6 measurement
        run: |
          python evals/evaluate.py
          python -m authorized_recall
      - name: the committed vectors match the generator
        run: |
          python conformance/generate.py
          git diff --exit-code conformance/