From efeb44f6439197cdb1abfedb96057b76ce322f24 Mon Sep 17 00:00:00 2001 From: nucli-vicky Date: Mon, 31 Aug 2026 14:50:30 +0200 Subject: [PATCH 1/2] skip other solvers when not changed --- .github/scripts/find_benchmarks.py | 49 ++++++++++++++++++++++++++++ .github/workflows/deepinv_run_hf.yml | 13 +++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/.github/scripts/find_benchmarks.py b/.github/scripts/find_benchmarks.py index 7c2b017..b20e6b6 100644 --- a/.github/scripts/find_benchmarks.py +++ b/.github/scripts/find_benchmarks.py @@ -3,11 +3,15 @@ import json import os +import re from pathlib import Path from git import Repo from git.exc import GitCommandError +# Matches e.g. ` name = "DRUNet"` inside a benchopt Solver class. +SOLVER_NAME_RE = re.compile(r"""^\s*name\s*=\s*(['"])(.*?)\1""", re.MULTILINE) + def find_benchmark_dirs(root: Path, max_depth: int = 4) -> list[str]: """Find all directories containing an objective.py file.""" @@ -81,6 +85,46 @@ def filter_changed_dirs(dirs: list[str], changed_files: set[str]) -> list[str]: ] +def parse_solver_name(path: Path) -> str | None: + """Extract the `name` class attribute from a benchopt solver file.""" + try: + text = path.read_text() + except OSError: + return None + match = SOLVER_NAME_RE.search(text) + return match.group(2) if match else None + + +def compute_solver_filters( + dirs: list[str], changed_files: set[str], root: Path +) -> dict[str, list[str]]: + """Compute, for each benchmark dir, which solvers to restrict a run to. + """ + filters: dict[str, list[str]] = {} + for d in dirs: + solver_prefix = d + "/solvers/" + dir_changed = {f for f in changed_files if f.startswith(d + "/")} + + if not dir_changed or any( + not f.startswith(solver_prefix) for f in dir_changed + ): + filters[d] = [] + continue + + names = [] + for f in dir_changed: + if not f.endswith(".py") or Path(f).name == "__init__.py": + continue + name = parse_solver_name(root / f) + if name is None: + names = [] + break + names.append(name) + + filters[d] = sorted(set(names)) + return filters + + def main() -> None: import argparse @@ -107,16 +151,20 @@ def main() -> None: "Valid values are:\n- " + "\n- ".join(all_dirs) ) filtered_dirs = [dispatch_benchmark_dir] + solver_filters = {} elif ref_range and not args.all: base, head = ref_range changed_files = get_changed_files(repo, base, head) filtered_dirs = filter_changed_dirs(all_dirs, changed_files) + solver_filters = compute_solver_filters(filtered_dirs, changed_files, root) else: # No ref_range (e.g., schedule/tag/create): include all benchmarks filtered_dirs = all_dirs + solver_filters = {} # Output as JSON print(f"Found benchmark directories:\n{filtered_dirs}") + print(f"Solver filters (empty list means run all solvers):\n{solver_filters}") result = json.dumps(filtered_dirs) # If running in GitHub Actions, set the output @@ -124,6 +172,7 @@ def main() -> None: if github_output: with open(github_output, "a") as f: f.write(f"dirs={result}\nfound_benchmarks={len(filtered_dirs) > 0}\n") + f.write(f"solver-filters={json.dumps(solver_filters)}\n") if __name__ == "__main__": diff --git a/.github/workflows/deepinv_run_hf.yml b/.github/workflows/deepinv_run_hf.yml index a3eb635..fc23162 100644 --- a/.github/workflows/deepinv_run_hf.yml +++ b/.github/workflows/deepinv_run_hf.yml @@ -19,6 +19,7 @@ jobs: outputs: benchmark-dirs: ${{ steps.find-dirs.outputs.dirs }} found_benchmarks: ${{ steps.find-dirs.outputs.found_benchmarks }} + solver-filters: ${{ steps.find-dirs.outputs.solver-filters }} steps: - uses: actions/checkout@v3 with: @@ -60,6 +61,7 @@ jobs: BENCHOPT_CONDA_CMD: 'mamba' BENCHOPT_RAISE_INSTALL_ERROR: true BENCHOPT_DEBUG: 1 + SOLVER_FILTERS_JSON: ${{ needs.find-benchmarks.outputs.solver-filters }} defaults: run: # Use non-login shell with BASH_ENV instead of -l to ensure @@ -110,8 +112,17 @@ jobs: --env-name ${{ env.RUN_CONDA_ENV }} - name: Run benchmarks + env: + BENCHMARK_DIR: ${{ matrix.benchmark_dir }} run: | - benchopt run ${{ matrix.benchmark_dir }} --output results_ci_run.csv \ + SOLVER_FLAGS=$(python3 -c " + import json, os, shlex + filters = json.loads(os.environ['SOLVER_FILTERS_JSON'] or '{}') + solvers = filters.get(os.environ['BENCHMARK_DIR'], []) + print(' '.join(f'-s {shlex.quote(name)}' for name in solvers)) + ") + benchopt run ${{ matrix.benchmark_dir }} $SOLVER_FLAGS \ + --output results_ci_run.csv \ --no-plot --env-name ${{ env.RUN_CONDA_ENV }} - name: Upload results From d795a2d53de9a39e16ff1fd48fb4d4a24f516146 Mon Sep 17 00:00:00 2001 From: nucli-vicky Date: Mon, 31 Aug 2026 15:20:24 +0200 Subject: [PATCH 2/2] lint --- .github/scripts/find_benchmarks.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/scripts/find_benchmarks.py b/.github/scripts/find_benchmarks.py index b20e6b6..bca1697 100644 --- a/.github/scripts/find_benchmarks.py +++ b/.github/scripts/find_benchmarks.py @@ -98,16 +98,13 @@ def parse_solver_name(path: Path) -> str | None: def compute_solver_filters( dirs: list[str], changed_files: set[str], root: Path ) -> dict[str, list[str]]: - """Compute, for each benchmark dir, which solvers to restrict a run to. - """ + """Compute, for each benchmark dir, which solvers to restrict a run to.""" filters: dict[str, list[str]] = {} for d in dirs: solver_prefix = d + "/solvers/" dir_changed = {f for f in changed_files if f.startswith(d + "/")} - if not dir_changed or any( - not f.startswith(solver_prefix) for f in dir_changed - ): + if not dir_changed or any(not f.startswith(solver_prefix) for f in dir_changed): filters[d] = [] continue