Run the scheduler constraint tests under both backends

flexmeasures/data/models/planning/tests/conftest.py

@@ -443,3 +443,68 @@ def add_as_beliefs(db, sensor, values, time_slots, source):
         for dt, val in zip(time_slots, values)
     ]
     db.session.add_all(beliefs)
+
+
+@pytest.fixture(autouse=True)
+def solver_backend(request, app):
+    """Run a test under a specific solver backend, when its module opts in.
+
+    Modules setting ``RUN_UNDER_EACH_SOLVER = True`` have every test run once per backend (see ``pytest_generate_tests`` below).
+    Everything else is untouched, and keeps running under the configured default.
+    """
+    solver = getattr(request, "param", None)
+    if solver is None:
+        yield None
+        return
+    original_solver = app.config["FLEXMEASURES_LP_SOLVER"]
+    app.config["FLEXMEASURES_LP_SOLVER"] = solver
+    yield solver
+    app.config["FLEXMEASURES_LP_SOLVER"] = original_solver
+
+
+def pytest_generate_tests(metafunc):
+    """Parametrize a whole module over the solver backends, if it opts in.
+
+    A module that exercises scheduler behaviour is only meaningful under one backend if the two agree,
+    which is exactly what we cannot assume:
+    the schedulers build the same model twice, once through Pyomo and once directly in HiGHS.
+    Opting a module in costs a signature change nowhere -- the autouse fixture above does the switching.
+    """
+    if getattr(metafunc.module, "RUN_UNDER_EACH_SOLVER", False):
+        metafunc.parametrize(
+            "solver_backend", ["appsi_highs", "highspy"], indirect=True
+        )
+
+
+def pytest_addoption(parser):
+    """Allow a whole run to be pinned to one solver backend.
+
+    The modules that cannot be parametrized in-process (see EXEMPT in test_solver_coverage.py)
+    can still be shown green under the other backend, by running them again with this flag.
+    Note that FLEXMEASURES_LP_SOLVER cannot be set from the environment for tests,
+    because TestingConfig does not read it.
+    """
+    parser.addoption(
+        "--lp-solver",
+        action="store",
+        default=None,
+        help="Run every test under this solver backend (e.g. appsi_highs).",
+    )
+
+
+@pytest.fixture(autouse=True)
+def pinned_solver_backend(request, app):
+    """Apply --lp-solver, unless the test is already parametrized over backends."""
+    solver = request.config.getoption("--lp-solver")
+    if (
+        solver is None
+        or "solver_backend" in request.fixturenames
+        and getattr(request.node, "callspec", None)
+        and "solver_backend" in request.node.callspec.params
+    ):
+        yield
+        return
+    original_solver = app.config["FLEXMEASURES_LP_SOLVER"]
+    app.config["FLEXMEASURES_LP_SOLVER"] = solver
+    yield
+    app.config["FLEXMEASURES_LP_SOLVER"] = original_solver

flexmeasures/data/models/planning/tests/test_group_constraints.py

@@ -10,6 +10,9 @@
 from flexmeasures.data.models.planning.storage import StorageScheduler
 from flexmeasures.utils.unit_utils import ur
 
+#: Run every test in this module under both scheduler backends (see conftest).
+RUN_UNDER_EACH_SOLVER = True
+
 
 def _unique_name(prefix: str) -> str:
     return f"{prefix} {uuid.uuid4().hex[:8]}"

flexmeasures/data/models/planning/tests/test_operation_modes.py

@@ -7,6 +7,9 @@
 from flexmeasures.data.models.planning.linear_optimization import device_scheduler
 from flexmeasures.data.models.planning.utils import initialize_index
 
+#: Run every test in this module under both scheduler backends (see conftest).
+RUN_UNDER_EACH_SOLVER = True
+
 
 def _one_device_setup(stock_target: float):
     """One storage device charging towards a stock target over 4 hourly steps.

flexmeasures/data/models/planning/tests/test_solver_coverage.py

@@ -0,0 +1,67 @@
+"""Keep track of which scheduler tests actually run under both backends.
+
+The schedulers build the same model twice — once through Pyomo, once directly in HiGHS —
+so a test of scheduler behaviour only means something under one backend if the two agree,
+which is the very thing that cannot be assumed.
+A module that does not opt into the solver matrix is therefore a coverage hole,
+and this test exists so that hole is an explicit, reviewed decision rather than an accident.
+
+To opt a module in, set ``RUN_UNDER_EACH_SOLVER = True`` at its top (see conftest).
+"""
+
+from __future__ import annotations
+
+import pathlib
+
+#: Modules that exercise scheduler behaviour but deliberately run under one solver only.
+#: Each needs a reason, and the reason should be fixable rather than permanent.
+EXEMPT = {
+    # These create assets with hardcoded names inline,
+    # so running each test twice in one fixture scope violates generic_asset's unique-name constraint.
+    # They still pass under the other backend when a whole run is pinned to it with --lp-solver.
+    "test_commitments.py": "creates named DB assets; not idempotent across parameters",
+    "test_storage.py": "creates named DB assets; not idempotent across parameters",
+    "test_process.py": "ProcessScheduler does not use device_scheduler",
+    # Covered by the solver matrix through their own fixture instead.
+    "test_solver.py": "uses the app_with_each_solver fixture directly",
+    "test_highspy_equivalence.py": "runs both backends explicitly, per scenario",
+    "test_solver_options.py": "tests option validation, not scheduling",
+    # No scheduling involved.
+    "test_device_inventory.py": "no scheduling",
+    "test_storage_utils.py": "no scheduling",
+    "test_utils.py": "no scheduling",
+    "test_utils_fresh_db.py": "no scheduling",
+}
+
+TESTS_DIR = pathlib.Path(__file__).parent
+
+
+def test_scheduler_modules_run_under_each_solver_or_are_exempt():
+    """Every planning test module either opts into the solver matrix or is listed as exempt.
+
+    If this fails after adding a module, decide which it is —
+    do not add it to EXEMPT just to go green.
+    """
+    uncovered = []
+    for path in sorted(TESTS_DIR.glob("test_*.py")):
+        if path.name == pathlib.Path(__file__).name:
+            continue
+        opts_in = "RUN_UNDER_EACH_SOLVER = True" in path.read_text()
+        if not opts_in and path.name not in EXEMPT:
+            uncovered.append(path.name)
+    assert not uncovered, (
+        "These planning test modules run under one solver only, and are not listed as exempt: "
+        f"{uncovered}. Either set RUN_UNDER_EACH_SOLVER = True, or add them to EXEMPT with a reason."
+    )
+
+
+def test_exempt_list_has_no_stale_entries():
+    """An exempt module that no longer exists, or that has since opted in, should be removed."""
+    stale = []
+    for name in EXEMPT:
+        path = TESTS_DIR / name
+        if not path.exists():
+            stale.append(f"{name} (gone)")
+        elif "RUN_UNDER_EACH_SOLVER = True" in path.read_text():
+            stale.append(f"{name} (now opts in)")
+    assert not stale, f"Stale EXEMPT entries: {stale}"