Do not run all solvers for a benchmark when only one is modified

.github/scripts/find_benchmarks.py

@@ -3,11 +3,15 @@
 
 import json
 import os
+import re
 from pathlib import Path
 
 from git import Repo
 from git.exc import GitCommandError
 
+# Matches e.g. `    name = "DRUNet"` inside a benchopt Solver class.
+SOLVER_NAME_RE = re.compile(r"""^\s*name\s*=\s*(['"])(.*?)\1""", re.MULTILINE)
+
 
 def find_benchmark_dirs(root: Path, max_depth: int = 4) -> list[str]:
     """Find all directories containing an objective.py file."""
@@ -81,6 +85,43 @@ def filter_changed_dirs(dirs: list[str], changed_files: set[str]) -> list[str]:
     ]
 
 
+def parse_solver_name(path: Path) -> str | None:
+    """Extract the `name` class attribute from a benchopt solver file."""
+    try:
+        text = path.read_text()
+    except OSError:
+        return None
+    match = SOLVER_NAME_RE.search(text)
+    return match.group(2) if match else None
+
+
+def compute_solver_filters(
+    dirs: list[str], changed_files: set[str], root: Path
+) -> dict[str, list[str]]:
+    """Compute, for each benchmark dir, which solvers to restrict a run to."""
+    filters: dict[str, list[str]] = {}
+    for d in dirs:
+        solver_prefix = d + "/solvers/"
+        dir_changed = {f for f in changed_files if f.startswith(d + "/")}
+
+        if not dir_changed or any(not f.startswith(solver_prefix) for f in dir_changed):
+            filters[d] = []
+            continue
+
+        names = []
+        for f in dir_changed:
+            if not f.endswith(".py") or Path(f).name == "__init__.py":
+                continue
+            name = parse_solver_name(root / f)
+            if name is None:
+                names = []
+                break
+            names.append(name)
+
+        filters[d] = sorted(set(names))
+    return filters
+
+
 def main() -> None:
 
     import argparse
@@ -107,23 +148,28 @@ def main() -> None:
             "Valid values are:\n- " + "\n- ".join(all_dirs)
         )
         filtered_dirs = [dispatch_benchmark_dir]
+        solver_filters = {}
     elif ref_range and not args.all:
         base, head = ref_range
         changed_files = get_changed_files(repo, base, head)
         filtered_dirs = filter_changed_dirs(all_dirs, changed_files)
+        solver_filters = compute_solver_filters(filtered_dirs, changed_files, root)
     else:
         # No ref_range (e.g., schedule/tag/create): include all benchmarks
         filtered_dirs = all_dirs
+        solver_filters = {}
 
     # Output as JSON
     print(f"Found benchmark directories:\n{filtered_dirs}")
+    print(f"Solver filters (empty list means run all solvers):\n{solver_filters}")
     result = json.dumps(filtered_dirs)
 
     # If running in GitHub Actions, set the output
     github_output = os.environ.get("GITHUB_OUTPUT")
     if github_output:
         with open(github_output, "a") as f:
             f.write(f"dirs={result}\nfound_benchmarks={len(filtered_dirs) > 0}\n")
+            f.write(f"solver-filters={json.dumps(solver_filters)}\n")
 
 
 if __name__ == "__main__":

.github/workflows/deepinv_run_hf.yml

@@ -19,6 +19,7 @@ jobs:
     outputs:
       benchmark-dirs: ${{ steps.find-dirs.outputs.dirs }}
       found_benchmarks: ${{ steps.find-dirs.outputs.found_benchmarks }}
+      solver-filters: ${{ steps.find-dirs.outputs.solver-filters }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -60,6 +61,7 @@ jobs:
         BENCHOPT_CONDA_CMD: 'mamba'
         BENCHOPT_RAISE_INSTALL_ERROR: true
         BENCHOPT_DEBUG: 1
+        SOLVER_FILTERS_JSON: ${{ needs.find-benchmarks.outputs.solver-filters }}
     defaults:
       run:
         # Use non-login shell with BASH_ENV instead of -l to ensure
@@ -110,8 +112,17 @@ jobs:
             --env-name ${{ env.RUN_CONDA_ENV }}
 
     - name: Run benchmarks
+      env:
+        BENCHMARK_DIR: ${{ matrix.benchmark_dir }}
       run: |
-        benchopt run ${{ matrix.benchmark_dir }} --output results_ci_run.csv \
+        SOLVER_FLAGS=$(python3 -c "
+        import json, os, shlex
+        filters = json.loads(os.environ['SOLVER_FILTERS_JSON'] or '{}')
+        solvers = filters.get(os.environ['BENCHMARK_DIR'], [])
+        print(' '.join(f'-s {shlex.quote(name)}' for name in solvers))
+        ")
+        benchopt run ${{ matrix.benchmark_dir }} $SOLVER_FLAGS \
+            --output results_ci_run.csv \
             --no-plot --env-name ${{ env.RUN_CONDA_ENV }}
 
     - name: Upload results