MNT update benchopt 1.9

.github/workflows/main.yml

@@ -11,8 +11,8 @@ on:
     branches:
       - main
   schedule:
-    # Run every day at 7:42am UTC.
-    - cron:  '42 7 * * *'
+    # Run first day of the month
+    - cron:  '42 7 1 * *'
 
 jobs:
   benchopt_dev:

datasets/py_bench.py

@@ -1,16 +1,14 @@
-from benchopt import safe_import_context
 from benchopt import BaseDataset
 
-with safe_import_context() as import_ctx:
-    from PyBenchFCN import SingleObjectiveProblem as SOP
+from PyBenchFCN import SingleObjectiveProblem as SOP
 
 
 class Dataset(BaseDataset):
 
     name = "FCN"
 
     install_cmd = "conda"
-    requirements = ["pip:PyBenchFCN"]
+    requirements = ["pip::PyBenchFCN"]
 
     # List of parameters to generate the datasets. The benchmark will consider
     # the cross product for each key in the dictionary.

datasets/simulated.py

@@ -1,28 +1,20 @@
-from benchopt import safe_import_context
 from benchopt import BaseDataset
 
-with safe_import_context() as import_ctx:
-    import numpy as np
+import numpy as np
 
 
 class Dataset(BaseDataset):
 
     name = "simulated"
-    install_cmd = "conda"
-    requirements = ["numpy"]
 
     # List of parameters to generate the datasets. The benchmark will consider
     # the cross product for each key in the dictionary.
     parameters = {
         "dimension": [2, 10],
     }
 
-    def __init__(self, dimension=2, bounds=(-3, 3)):
-        self.function = lambda x: np.linalg.norm(x, 2) ** 2
-        self.dimension = dimension
-        self.bounds = bounds
-
     def get_data(self):
-        return dict(function=self.function,
-                    dimension=self.dimension,
-                    bounds=self.bounds)
+        return dict(
+            function=lambda x: np.linalg.norm(x, 2) ** 2,
+            dimension=self.dimension, bounds=(-3, 3)
+        )

objective.py

@@ -1,25 +1,22 @@
-from benchopt import BaseObjective, safe_import_context
+from benchopt import BaseObjective
 
-# Protect import to allow manipulating objective without importing library
-# Useful for autocompletion and install commands
-with safe_import_context() as import_ctx:
-    import numpy as np
+import numpy as np
 
 
 class Objective(BaseObjective):
-    min_benchopt_version = "1.3"
+    min_benchopt_version = "1.9"
     name = "Zero-order test functions"
 
-    def get_one_solution(self):
-        # Return one solution. This should be compatible with 'self.compute'.
-        return np.zeros(self.dimension)
+    def get_one_result(self):
+        # Return one result for testing purpose.
+        return dict(x=np.zeros(self.dimension))
 
     def set_data(self, function, dimension, bounds):
         self.function = function
         self.dimension = dimension
         self.bounds = bounds
 
-    def compute(self, x):
+    def evaluate_result(self, x):
         return self.function(x)
 
     def get_objective(self):

solvers/basinhopping.py

@@ -11,10 +11,9 @@ class Solver(BaseSolver):
     name = "basinhopping"
 
     install_cmd = "conda"
-    requirements = ["numpy", "scipy"]
+    requirements = ["scipy"]
     parameters = {
         "temperature": [1, 10],
-        "seed": [42],
     }
 
     def set_objective(self, function, dimension, bounds):
@@ -24,7 +23,10 @@ def set_objective(self, function, dimension, bounds):
 
     def run(self, n_iter):
         f = self.function
-        rng = np.random.RandomState(self.seed)  # fix seed
+        seed = self.get_seed(
+            use_repetition=True, use_dataset=True, use_solver=True
+        )
+        rng = np.random.RandomState(seed)  # fix seed
         x0 = rng.uniform(size=self.dimension,
                          low=self.bounds[0],
                          high=self.bounds[1])
@@ -35,4 +37,4 @@ def run(self, n_iter):
         self.xopt = result.x
 
     def get_result(self):
-        return self.xopt.flatten()
+        return dict(x=self.xopt.flatten())

solvers/nevergrad.py

@@ -1,8 +1,7 @@
-from benchopt import BaseSolver, safe_import_context
+from benchopt import BaseSolver
 
-with safe_import_context() as import_ctx:
-    import numpy as np
-    import nevergrad as ng
+import numpy as np
+import nevergrad as ng
 
 
 class Solver(BaseSolver):
@@ -11,39 +10,53 @@ class Solver(BaseSolver):
     name = "nevergrad"
 
     install_cmd = "conda"
-    requirements = [
-        "nevergrad",
-    ]
+    requirements = ["pip::nevergrad"]
     parameters = {
         "solver": ["NGOpt", "RandomSearch", "ScrHammersleySearch",
                    "TwoPointsDE", "CMA", "PSO"],
-        "seed": [42],
     }
 
+    sampling_strategy = 'callback'
+
     def set_objective(self, function, dimension, bounds):
         self.function = function
         self.dimension = dimension
         self.bounds = bounds
 
-    def run(self, n_iter):
-        rng = np.random.RandomState(self.seed)  # fix seed
+    def run(self, cb):
+        f = self.function
+        self.xopt = None
 
-        if n_iter == 0:
-            x0 = rng.uniform(size=self.dimension,
-                             low=self.bounds[0],
-                             high=self.bounds[1])
-            self.xopt = x0
-            return
+        # Get a seed that varies across repetitions, datasets and solvers,
+        # to ensure a good coverage of the search space, while still being
+        # reproducible.
+        seed = self.get_seed(
+            use_repetition=True, use_dataset=True, use_solver=True
+        )
+        rng = np.random.RandomState(seed)
 
-        f = self.function
         parametrization = ng.p.Array(shape=(self.dimension,))
         parametrization.set_bounds(self.bounds[0], self.bounds[1])
         parametrization.random_state = rng  # fix seed
         optimizer = ng.optimizers.registry[self.solver](
-            budget=n_iter, parametrization=parametrization, num_workers=1
+            budget=1000, parametrization=parametrization, num_workers=1
         )
+
+        def stop_criterion(optimizer):
+            if optimizer.num_tell == 0:
+                return False
+
+            recommendation = optimizer.provide_recommendation()
+            if recommendation is not None:
+                self.xopt = np.asarray(recommendation.value).flatten()
+            return not cb()
+
+        optimizer.register_callback("ask", ng.callbacks.EarlyStopping(
+            stop_criterion
+        ))
+
         recommendation = optimizer.minimize(f)
-        self.xopt = np.array(recommendation.value)
+        self.xopt = np.asarray(recommendation.value).flatten()
 
     def get_result(self):
-        return self.xopt.flatten()
+        return dict(x=self.xopt)

solvers/optuna.py

@@ -1,10 +1,10 @@
-from benchopt import BaseSolver, safe_import_context
-from benchopt.stopping_criterion import SufficientProgressCriterion
+from benchopt import BaseSolver
 
-with safe_import_context() as import_ctx:
-    import numpy as np
-    import optuna
-    from optuna import samplers
+import numpy as np
+import optuna
+from optuna import samplers
+# Check that cmaes is installed
+import cmaes  # noqa: F401
 
 
 class Solver(BaseSolver):
@@ -13,35 +13,40 @@ class Solver(BaseSolver):
     name = "optuna"
 
     install_cmd = "conda"
-    requirements = [
-        "optuna",
-        "cmaes",
-        "numpy",
-    ]
+    requirements = ["optuna", "cmaes"]
     parameters = {
-        "solver": ["cmaes", "TPE", "RandomSearch"],
-        "seed": [42],
+        "solver": ["cmaes", "TPE", "RandomSearch"]
     }
 
-    stopping_criterion = SufficientProgressCriterion(
-        patience=3, strategy='iteration')
+    sampling_strategy = 'callback'
 
     def set_objective(self, function, dimension, bounds):
         self.function = function
         self.dimension = dimension
         self.bounds = bounds
 
-    def run(self, n_iter):
-        n_iter += 1  # no possible to call optuna with 0 trial
-
+    def run(self, cb):
         def objective(trial):
             x = np.array([
                 trial.suggest_float(f'x_{k}', self.bounds[0], self.bounds[1])
                 for k in range(self.dimension)
             ])
             return self.function(x)
 
-        seed = self.seed  # to make results reproducible
+        class StopCallback:
+
+            def __call__(self, study, trial):
+                # Call the callback after each function evaluation, and stop
+                # the optimization if the callback returns False.
+                if not cb():
+                    study.stop()
+
+        # Get a seed that varies across repetitions, datasets and solvers,
+        # to ensure a good coverage of the search space, while still being
+        # reproducible.
+        seed = self.get_seed(
+            use_repetition=True, use_dataset=True, use_solver=True
+        )
         if self.solver == "TPE":
             sampler = samplers.TPESampler(seed=seed, n_startup_trials=10)
         elif self.solver == "RandomSearch":
@@ -50,11 +55,16 @@ def objective(trial):
             sampler = samplers.CmaEsSampler(seed=seed)
         else:
             raise NotImplementedError(f"Solver {self.solver} not implemented")
-        study = optuna.create_study(sampler=sampler, direction='minimize')
+        self.study_ = optuna.create_study(
+            sampler=sampler, direction='minimize'
+        )
         optuna.logging.disable_default_handler()  # limit verbosity
-        study.optimize(objective, n_trials=n_iter)
-        self.xopt = study.best_trial.params
+        self.study_.optimize(
+            objective, n_trials=1000, callbacks=[StopCallback()]
+        )
+        self.xopt = self.study_.best_trial.params
 
     def get_result(self):
-        xopt = np.array([self.xopt[f'x_{k}'] for k in range(self.dimension)])
-        return xopt
+        best_param = self.study_.best_trial.params
+        xopt = np.array([best_param[f'x_{k}'] for k in range(self.dimension)])
+        return dict(x=xopt)

solvers/scipy.py

@@ -1,8 +1,7 @@
-from benchopt import BaseSolver, safe_import_context
+from benchopt import BaseSolver
 
-with safe_import_context() as import_ctx:
-    import numpy as np
-    from scipy.optimize import minimize
+import numpy as np
+from scipy.optimize import minimize
 
 
 class Solver(BaseSolver):
@@ -14,32 +13,57 @@ class Solver(BaseSolver):
     requirements = ["numpy", "scipy"]
     parameters = {
         "solver": ["Nelder-Mead", "Powell", "BFGS"],
-        "seed": [42],
     }
 
+    sampling_strategy = 'callback'
+
     def set_objective(self, function, dimension, bounds):
         self.function = function
         self.dimension = dimension
         self.bounds = bounds
 
-    def run(self, n_iter):
+    def run(self, cb):
         f = self.function
-        rng = np.random.RandomState(self.seed)  # fix seed
+
+        seed = self.get_seed(
+            use_repetition=True, use_dataset=True, use_solver=True
+        )
+        rng = np.random.RandomState(seed)  # fix seed
         x0 = rng.uniform(size=self.dimension,
                          low=self.bounds[0],
                          high=self.bounds[1])
+        self.xopt = x0
+        best_val = np.inf
 
-        if n_iter == 0:
-            self.xopt = x0
-            return
+        class _StopScipy(Exception):
+            pass
 
-        result = minimize(
-            f,
-            x0=x0,
-            method=self.solver,
-            options={"maxiter": n_iter, "xatol": 1e-20, "fatol": 1e-20},
-        )
-        self.xopt = result.x
+        def objective(x):
+            nonlocal best_val
+            value = f(x)
+            if value < best_val:
+                best_val = value
+                self.xopt = np.asarray(x).flatten()
+            return value
+
+        def scipy_callback(xk):
+            if not cb():
+                raise _StopScipy()
+
+        options = {}
+        if self.solver in ("Nelder-Mead", "Powell"):
+            options.update({"xatol": 1e-20, "fatol": 1e-20})
+
+        try:
+            minimize(
+                objective,
+                x0=x0,
+                method=self.solver,
+                callback=scipy_callback,
+                options=options,
+            )
+        except _StopScipy:
+            pass
 
     def get_result(self):
-        return self.xopt.flatten()
+        return dict(x=self.xopt)