diff --git a/flexmeasures/data/models/planning/tests/conftest.py b/flexmeasures/data/models/planning/tests/conftest.py index 17752bdabe..1d0b54fa2a 100644 --- a/flexmeasures/data/models/planning/tests/conftest.py +++ b/flexmeasures/data/models/planning/tests/conftest.py @@ -443,3 +443,68 @@ def add_as_beliefs(db, sensor, values, time_slots, source): for dt, val in zip(time_slots, values) ] db.session.add_all(beliefs) + + +@pytest.fixture(autouse=True) +def solver_backend(request, app): + """Run a test under a specific solver backend, when its module opts in. + + Modules setting ``RUN_UNDER_EACH_SOLVER = True`` have every test run once per backend (see ``pytest_generate_tests`` below). + Everything else is untouched, and keeps running under the configured default. + """ + solver = getattr(request, "param", None) + if solver is None: + yield None + return + original_solver = app.config["FLEXMEASURES_LP_SOLVER"] + app.config["FLEXMEASURES_LP_SOLVER"] = solver + yield solver + app.config["FLEXMEASURES_LP_SOLVER"] = original_solver + + +def pytest_generate_tests(metafunc): + """Parametrize a whole module over the solver backends, if it opts in. + + A module that exercises scheduler behaviour is only meaningful under one backend if the two agree, + which is exactly what we cannot assume: + the schedulers build the same model twice, once through Pyomo and once directly in HiGHS. + Opting a module in costs a signature change nowhere -- the autouse fixture above does the switching. + """ + if getattr(metafunc.module, "RUN_UNDER_EACH_SOLVER", False): + metafunc.parametrize( + "solver_backend", ["appsi_highs", "highspy"], indirect=True + ) + + +def pytest_addoption(parser): + """Allow a whole run to be pinned to one solver backend. + + The modules that cannot be parametrized in-process (see EXEMPT in test_solver_coverage.py) + can still be shown green under the other backend, by running them again with this flag. + Note that FLEXMEASURES_LP_SOLVER cannot be set from the environment for tests, + because TestingConfig does not read it. + """ + parser.addoption( + "--lp-solver", + action="store", + default=None, + help="Run every test under this solver backend (e.g. appsi_highs).", + ) + + +@pytest.fixture(autouse=True) +def pinned_solver_backend(request, app): + """Apply --lp-solver, unless the test is already parametrized over backends.""" + solver = request.config.getoption("--lp-solver") + if ( + solver is None + or "solver_backend" in request.fixturenames + and getattr(request.node, "callspec", None) + and "solver_backend" in request.node.callspec.params + ): + yield + return + original_solver = app.config["FLEXMEASURES_LP_SOLVER"] + app.config["FLEXMEASURES_LP_SOLVER"] = solver + yield + app.config["FLEXMEASURES_LP_SOLVER"] = original_solver diff --git a/flexmeasures/data/models/planning/tests/test_group_constraints.py b/flexmeasures/data/models/planning/tests/test_group_constraints.py index 88f8e119c7..4d90f26531 100644 --- a/flexmeasures/data/models/planning/tests/test_group_constraints.py +++ b/flexmeasures/data/models/planning/tests/test_group_constraints.py @@ -10,6 +10,9 @@ from flexmeasures.data.models.planning.storage import StorageScheduler from flexmeasures.utils.unit_utils import ur +#: Run every test in this module under both scheduler backends (see conftest). +RUN_UNDER_EACH_SOLVER = True + def _unique_name(prefix: str) -> str: return f"{prefix} {uuid.uuid4().hex[:8]}" diff --git a/flexmeasures/data/models/planning/tests/test_operation_modes.py b/flexmeasures/data/models/planning/tests/test_operation_modes.py index aaa6a3a7e4..3ccf64fc8f 100644 --- a/flexmeasures/data/models/planning/tests/test_operation_modes.py +++ b/flexmeasures/data/models/planning/tests/test_operation_modes.py @@ -7,6 +7,9 @@ from flexmeasures.data.models.planning.linear_optimization import device_scheduler from flexmeasures.data.models.planning.utils import initialize_index +#: Run every test in this module under both scheduler backends (see conftest). +RUN_UNDER_EACH_SOLVER = True + def _one_device_setup(stock_target: float): """One storage device charging towards a stock target over 4 hourly steps. diff --git a/flexmeasures/data/models/planning/tests/test_solver_coverage.py b/flexmeasures/data/models/planning/tests/test_solver_coverage.py new file mode 100644 index 0000000000..18bc5ef1ed --- /dev/null +++ b/flexmeasures/data/models/planning/tests/test_solver_coverage.py @@ -0,0 +1,67 @@ +"""Keep track of which scheduler tests actually run under both backends. + +The schedulers build the same model twice — once through Pyomo, once directly in HiGHS — +so a test of scheduler behaviour only means something under one backend if the two agree, +which is the very thing that cannot be assumed. +A module that does not opt into the solver matrix is therefore a coverage hole, +and this test exists so that hole is an explicit, reviewed decision rather than an accident. + +To opt a module in, set ``RUN_UNDER_EACH_SOLVER = True`` at its top (see conftest). +""" + +from __future__ import annotations + +import pathlib + +#: Modules that exercise scheduler behaviour but deliberately run under one solver only. +#: Each needs a reason, and the reason should be fixable rather than permanent. +EXEMPT = { + # These create assets with hardcoded names inline, + # so running each test twice in one fixture scope violates generic_asset's unique-name constraint. + # They still pass under the other backend when a whole run is pinned to it with --lp-solver. + "test_commitments.py": "creates named DB assets; not idempotent across parameters", + "test_storage.py": "creates named DB assets; not idempotent across parameters", + "test_process.py": "ProcessScheduler does not use device_scheduler", + # Covered by the solver matrix through their own fixture instead. + "test_solver.py": "uses the app_with_each_solver fixture directly", + "test_highspy_equivalence.py": "runs both backends explicitly, per scenario", + "test_solver_options.py": "tests option validation, not scheduling", + # No scheduling involved. + "test_device_inventory.py": "no scheduling", + "test_storage_utils.py": "no scheduling", + "test_utils.py": "no scheduling", + "test_utils_fresh_db.py": "no scheduling", +} + +TESTS_DIR = pathlib.Path(__file__).parent + + +def test_scheduler_modules_run_under_each_solver_or_are_exempt(): + """Every planning test module either opts into the solver matrix or is listed as exempt. + + If this fails after adding a module, decide which it is — + do not add it to EXEMPT just to go green. + """ + uncovered = [] + for path in sorted(TESTS_DIR.glob("test_*.py")): + if path.name == pathlib.Path(__file__).name: + continue + opts_in = "RUN_UNDER_EACH_SOLVER = True" in path.read_text() + if not opts_in and path.name not in EXEMPT: + uncovered.append(path.name) + assert not uncovered, ( + "These planning test modules run under one solver only, and are not listed as exempt: " + f"{uncovered}. Either set RUN_UNDER_EACH_SOLVER = True, or add them to EXEMPT with a reason." + ) + + +def test_exempt_list_has_no_stale_entries(): + """An exempt module that no longer exists, or that has since opted in, should be removed.""" + stale = [] + for name in EXEMPT: + path = TESTS_DIR / name + if not path.exists(): + stale.append(f"{name} (gone)") + elif "RUN_UNDER_EACH_SOLVER = True" in path.read_text(): + stale.append(f"{name} (now opts in)") + assert not stale, f"Stale EXEMPT entries: {stale}"