diff --git a/AGENTS.md b/AGENTS.md index 663e57b8..eef978bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,7 +72,7 @@ The codebase is split into three packages along a solver-dependency boundary: **`gems_craft/model/`** — Immutable model templates. - `Model`: defines component behavior (parameters, variables, constraints, ports) - `Library`: a collection of models, loaded from YAML -- `Taxonomy` (`taxonomy.py`): categories naming the items a model must expose. Models opt in via `taxonomy-category`; `check_library_against_taxonomy` enforces conformance. +- `Taxonomy` (`taxonomy.py`): categories naming the items a model must expose. Models opt in via `taxonomy-category`. `taxonomy.py` holds the data and `load_taxonomy` only; conformance lives in `validation.py` (`check_library_against_taxonomy`, and `validate_libraries_against_taxonomy` for libraries declaring a `taxonomy`), which `load_study` calls after parsing — `parse_yaml_library` stays a pure reader. `load_study` reads the `Taxonomy` from the optional `input/taxonomy.yml`. - `PortTypeSchema` (`parsing.py`) also declares the hybrid-only `area-connection` (`AreaConnectionSchema`) and `thermal-capacity-connection` (`PortThermalCapacitySchema`) fields; `resolve_library.py`'s `_convert_port_type` parses and discards them for every library, hybrid or not. **`gems_craft/expression/`** — Mathematical expression language and AST (structural/static analysis only — no numeric evaluation). diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ade1c908..022be60c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,6 +4,30 @@ All notable changes to GemsPy are documented here. ## [Unreleased] +### Added +- **Taxonomy conformance checked when a study is loaded** - `load_study` reads + the optional `input/taxonomy.yml` and calls + `validate_libraries_against_taxonomy` on every library declaring a `taxonomy` + field. `parse_yaml_library` is unchanged and performs no validation. + +### Changed +- **Breaking** - loading a study whose library declares a `taxonomy` raises + `ValueError` if no taxonomy is supplied, or if its id differs from the declared + one. +- **Breaking** - `TaxonomyData` renamed to `TaxonomySchema`; no alias kept. +- **Breaking** - `check_library_against_taxonomy` moved from + `gems_craft.model.taxonomy` to the new `gems_craft.model.validation`, and + `consistency_check` from `gems_craft.study.resolve_components` to the new + `gems_craft.study.validation`, where it is renamed + `check_component_models`. The `Study.check_consistency()` method moved to + that same module as the function `check_data_requirements(study)` — the two + checks are disjoint (component model ids vs. database coverage) and their + near-identical old names invited confusion. Reading and validating are now + separate modules, as in `optim_config/`. No behavior change; names and + import paths only. + +--- + ## [0.2.0] - 2026-08-24 ### Added diff --git a/docs/getting-started.md b/docs/getting-started.md index 7a5dd277..8bdca90a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -18,7 +18,7 @@ The first category of input files mentioned above comprises libraries of models. library: id: basic description: Basic library - taxonomy: basic_taxonomy_of_models # optional: id of the taxonomy this library conforms to + taxonomy: basic_taxonomy_of_models # optional: id of the taxonomy this library is checked against version: 1.0.0 # optional: free-form library version port-types: diff --git a/docs/user-guide/inputs.md b/docs/user-guide/inputs.md index f50646f4..efd8f731 100644 --- a/docs/user-guide/inputs.md +++ b/docs/user-guide/inputs.md @@ -14,6 +14,7 @@ my_study/ ├── input/ │ ├── system.yml │ ├── optim-config.yml +│ ├── taxonomy.yml ← optional │ ├── model-libraries/ │ │ └── *.yml │ └── data-series/ diff --git a/src/gems_craft/model/taxonomy.py b/src/gems_craft/model/taxonomy.py index eba774b3..b2eb2410 100644 --- a/src/gems_craft/model/taxonomy.py +++ b/src/gems_craft/model/taxonomy.py @@ -12,12 +12,11 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import Callable, Dict, List, Optional +from typing import List, Optional import yaml from pydantic import Field -from gems_craft.model.parsing import LibrarySchema, ModelSchema from gems_craft.utils import ModifiedBaseModel @@ -38,7 +37,7 @@ class TaxonomyCategory(ModifiedBaseModel): properties: List[TaxonomyItem] = Field(default_factory=list) -class TaxonomyData(ModifiedBaseModel): +class TaxonomySchema(ModifiedBaseModel): id: str description: str = "" categories: List[TaxonomyCategory] = Field(default_factory=list) @@ -56,90 +55,7 @@ def load_taxonomy(taxonomy_file: Path) -> Taxonomy: raw = yaml.safe_load(f) if "taxonomy" not in raw: raise ValueError(f"Missing 'taxonomy' key at root of {taxonomy_file}") - data = TaxonomyData.model_validate(raw["taxonomy"]) + data = TaxonomySchema.model_validate(raw["taxonomy"]) return Taxonomy( id=data.id, description=data.description, categories=data.categories ) - - -def _missing( - required: List[TaxonomyItem], exposed: List, exposed_key: Callable -) -> List[str]: - """Return the sorted taxonomy item ids not exposed by the model. - - Taxonomy items are always identified by their ``id``; ``exposed_key`` maps each - model-side item to the identifier to compare against (e.g. the ``port.field`` - string for port-field-definitions). - """ - return sorted( - {item.id for item in required} - {exposed_key(item) for item in exposed} - ) - - -def check_library_against_taxonomy(library: LibrarySchema, taxonomy: Taxonomy) -> None: - """ - Validates that every model declaring a taxonomy_category: - 1. References a category that exists in the taxonomy. - 2. Exposes all variables, parameters, ports, port-field-definitions, - constraints, binding-constraints, extra-outputs and properties listed - in that taxonomy category. - - Raises ValueError describing the first violation found. - """ - categories: Dict[str, TaxonomyCategory] = {c.id: c for c in taxonomy.categories} - - by_id: Callable = lambda x: x.id - - # Each entry maps a human-readable field-group name to the required items - # (from the taxonomy category) and the items exposed by the model, plus the - # function identifying a model-side item within that group. Taxonomy items are - # homogeneous (``TaxonomyItem``) and always identified by their ``id``. - def field_groups( - category: TaxonomyCategory, model_schema: "ModelSchema" - ) -> List[tuple]: - port_field_key: Callable = lambda d: f"{d.port}.{d.field}" - return [ - ("variable", category.variables, model_schema.variables, by_id), - ("parameter", category.parameters, model_schema.parameters, by_id), - ("port", category.ports, model_schema.ports, by_id), - ( - "port-field-definition", - category.port_field_definitions, - model_schema.port_field_definitions, - port_field_key, - ), - ("constraint", category.constraints, model_schema.constraints, by_id), - ( - "binding-constraint", - category.binding_constraints, - model_schema.binding_constraints, - by_id, - ), - ( - "extra-output", - category.extra_outputs, - model_schema.extra_outputs or [], - by_id, - ), - ("property", category.properties, model_schema.properties, by_id), - ] - - for model_schema in library.models: - cat_id = model_schema.taxonomy_category - if cat_id is None: - continue - - if cat_id not in categories: - raise ValueError( - f"Model '{model_schema.id}' references taxonomy category '{cat_id}' " - f"which does not exist in taxonomy '{taxonomy.id}'." - ) - - category = categories[cat_id] - for group_name, required, exposed, key in field_groups(category, model_schema): - missing = _missing(required, exposed, key) - if missing: - raise ValueError( - f"Model '{model_schema.id}' (taxonomy-category: '{cat_id}') is " - f"missing {group_name}(s) required by the taxonomy: {missing}." - ) diff --git a/src/gems_craft/model/validation.py b/src/gems_craft/model/validation.py new file mode 100644 index 00000000..6a6ee951 --- /dev/null +++ b/src/gems_craft/model/validation.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026, RTE (https://www.rte-france.com) +# +# See AUTHORS.txt +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# SPDX-License-Identifier: MPL-2.0 +# +# This file is part of the Antares project. + +"""Cross-validation of parsed model libraries. + +Kept apart from `parsing.py` (which only reads YAML into schemas) and from +`taxonomy.py` (which only holds the taxonomy data and reads it from disk), so +that reading and validating stay separate concerns — mirroring +`optim_config/parsing.py` and `optim_config/validation.py`. +""" + +from typing import Callable, Dict, List, Optional + +from gems_craft.model.parsing import LibrarySchema, ModelSchema +from gems_craft.model.taxonomy import Taxonomy, TaxonomyCategory, TaxonomyItem + + +def validate_libraries_against_taxonomy( + libraries: List[LibrarySchema], taxonomy: Optional[Taxonomy] +) -> None: + """Check every library declaring a ``taxonomy`` field against ``taxonomy``. + + Libraries that do not declare a taxonomy are left alone, even when some of + their models carry a ``taxonomy-category``. + + Raises ValueError if a declaring library has no taxonomy to check against, + if it declares a different taxonomy id, or if any of its models violates it. + """ + for library in libraries: + if library.taxonomy is None: + continue + declared = f"Library '{library.id}' declares taxonomy '{library.taxonomy}'" + if taxonomy is None: + raise ValueError( + f"{declared} but no taxonomy was provided to check it against." + ) + if library.taxonomy != taxonomy.id: + raise ValueError(f"{declared} but was checked against '{taxonomy.id}'.") + check_library_against_taxonomy(library, taxonomy) + + +def _missing( + required: List[TaxonomyItem], exposed: List, exposed_key: Callable +) -> List[str]: + """Return the sorted taxonomy item ids not exposed by the model. + + Taxonomy items are always identified by their ``id``; ``exposed_key`` maps each + model-side item to the identifier to compare against (e.g. the ``port.field`` + string for port-field-definitions). + """ + return sorted( + {item.id for item in required} - {exposed_key(item) for item in exposed} + ) + + +def check_library_against_taxonomy(library: LibrarySchema, taxonomy: Taxonomy) -> None: + """ + Validates that every model declaring a taxonomy_category: + 1. References a category that exists in the taxonomy. + 2. Exposes all variables, parameters, ports, port-field-definitions, + constraints, binding-constraints, extra-outputs and properties listed + in that taxonomy category. + + Raises ValueError describing the first violation found. + """ + categories: Dict[str, TaxonomyCategory] = {c.id: c for c in taxonomy.categories} + + by_id: Callable = lambda x: x.id + + # Each entry maps a human-readable field-group name to the required items + # (from the taxonomy category) and the items exposed by the model, plus the + # function identifying a model-side item within that group. Taxonomy items are + # homogeneous (``TaxonomyItem``) and always identified by their ``id``. + def field_groups( + category: TaxonomyCategory, model_schema: ModelSchema + ) -> List[tuple]: + port_field_key: Callable = lambda d: f"{d.port}.{d.field}" + return [ + ("variable", category.variables, model_schema.variables, by_id), + ("parameter", category.parameters, model_schema.parameters, by_id), + ("port", category.ports, model_schema.ports, by_id), + ( + "port-field-definition", + category.port_field_definitions, + model_schema.port_field_definitions, + port_field_key, + ), + ("constraint", category.constraints, model_schema.constraints, by_id), + ( + "binding-constraint", + category.binding_constraints, + model_schema.binding_constraints, + by_id, + ), + ( + "extra-output", + category.extra_outputs, + model_schema.extra_outputs or [], + by_id, + ), + ("property", category.properties, model_schema.properties, by_id), + ] + + for model_schema in library.models: + cat_id = model_schema.taxonomy_category + if cat_id is None: + continue + + if cat_id not in categories: + raise ValueError( + f"Model '{model_schema.id}' references taxonomy category '{cat_id}' " + f"which does not exist in taxonomy '{taxonomy.id}'." + ) + + category = categories[cat_id] + for group_name, required, exposed, key in field_groups(category, model_schema): + missing = _missing(required, exposed, key) + if missing: + raise ValueError( + f"Model '{model_schema.id}' (taxonomy-category: '{cat_id}') is " + f"missing {group_name}(s) required by the taxonomy: {missing}." + ) diff --git a/src/gems_craft/study/folder.py b/src/gems_craft/study/folder.py index e006b8c9..5ef9d306 100644 --- a/src/gems_craft/study/folder.py +++ b/src/gems_craft/study/folder.py @@ -5,6 +5,8 @@ - `input/system.yml`: A file describing the system to be simulated. - `input/model-libraries/`: A folder containing model library files in YAML format. - `input/data-series/`: A folder containing data series files. +- `input/taxonomy.yml` (optional): A taxonomy that libraries declaring a `taxonomy` + field are checked against. """ from pathlib import Path @@ -12,14 +14,16 @@ from gems_craft.model.model import Model from gems_craft.model.parsing import parse_yaml_library from gems_craft.model.resolve_library import resolve_library +from gems_craft.model.taxonomy import load_taxonomy +from gems_craft.model.validation import validate_libraries_against_taxonomy from gems_craft.study.parsing import parse_yaml_system from gems_craft.study.resolve_components import ( build_data_base, - consistency_check, resolve_system, ) from gems_craft.study.scenario_builder import ScenarioBuilder from gems_craft.study.study import Study +from gems_craft.study.validation import check_component_models def load_study(study_dir: Path) -> Study: @@ -28,7 +32,8 @@ def load_study(study_dir: Path) -> Study: This function reads the system definition, model libraries, and data series from the study directory, resolves them, and builds the simulation system - and database. + and database. If `input/taxonomy.yml` exists, every library declaring a + `taxonomy` is checked against it. Args: study_dir: The path to the study directory. @@ -39,11 +44,15 @@ def load_study(study_dir: Path) -> Study: system_file = study_dir / "input" / "system.yml" lib_folder = study_dir / "input" / "model-libraries" series_dir = study_dir / "input" / "data-series" + taxonomy_file = study_dir / "input" / "taxonomy.yml" + + taxonomy = load_taxonomy(taxonomy_file) if taxonomy_file.exists() else None input_libraries = [] for lib_file in lib_folder.glob("*.yml"): with lib_file.open() as lib: input_libraries.append(parse_yaml_library(lib)) + validate_libraries_against_taxonomy(input_libraries, taxonomy) with system_file.open() as c: input_study = parse_yaml_system(c) @@ -52,7 +61,7 @@ def load_study(study_dir: Path) -> Study: model_dict: dict[str, Model] = {} for library in lib_dict.values(): model_dict |= library.models - consistency_check(system, model_dict) + check_component_models(system, model_dict) scenario_builder_path = ( study_dir / "input" / "data-series" / "modeler-scenariobuilder.dat" diff --git a/src/gems_craft/study/resolve_components.py b/src/gems_craft/study/resolve_components.py index ced3457b..83197607 100644 --- a/src/gems_craft/study/resolve_components.py +++ b/src/gems_craft/study/resolve_components.py @@ -12,7 +12,6 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple, Union -from gems_craft.model import Model from gems_craft.model.library import Library from gems_craft.study import ( Component, @@ -132,23 +131,6 @@ def _get_component_by_id( return components_dict.get(component_id) -# TODO: cross-validation logic mixed into a "resolve" (preprocessing) module — -# move to a dedicated gems_craft/study/validation.py, mirroring optim_config/. -def consistency_check(system: System, input_models: Dict[str, Model]) -> bool: - """ - Checks if all components in the System have a valid model from the library. - Returns True if all components are consistent, raises ValueError otherwise. - """ - # TODO: Update this consistency check to check if each component have a valid model from the lib it refers to (and not all libs) - model_ids_set = input_models.keys() - for component in system.all_components: - if component.model.id not in model_ids_set: - raise ValueError( - f"Error: Component {component.id} has invalid model ID: {component.model.id}" - ) - return True - - def build_data_base( input_system: SystemSchema, timeseries_dir: Optional[Path], diff --git a/src/gems_craft/study/study.py b/src/gems_craft/study/study.py index 32080be0..431ebd46 100644 --- a/src/gems_craft/study/study.py +++ b/src/gems_craft/study/study.py @@ -28,10 +28,9 @@ class Study: DataBase (parameter values for those components). These two objects are always used together to build an optimisation - problem. ``Study`` gathers them into a single, coherent unit and - provides the cross-validation logic that was previously spread between - ``DataBase.requirements_consistency`` and the callers of - ``build_problem``. + problem. ``Study`` gathers them into a single, coherent unit; the + cross-validation of the pair lives in ``study/validation.py`` + (``check_data_requirements``). """ system: System @@ -52,29 +51,3 @@ def models(self) -> Dict[str, Model]: return { mk: components[0].model for mk, components in self.model_components.items() } - - # TODO: this is a second, disjoint consistency check alongside - # resolve_components.consistency_check() — consider consolidating both - # into one validation module for System/Study. - def check_consistency(self) -> None: - """Validate that the database supplies data for every parameter of every - component defined in the system. - - Raises - ------ - ValueError - If a required data entry is missing or its time/scenario structure - does not match what the model parameter expects. - """ - for component in self.system.components: - for param in component.model.parameters.values(): - data_structure = self.database.get_data(component.id, param.name) - - if not data_structure.check_requirement( - component.model.parameters[param.name].structure.time, - component.model.parameters[param.name].structure.scenario, - ): - raise ValueError( - f"Data inconsistency for component: {component.id}, " - f"parameter: {param.name}. Requirement not met." - ) diff --git a/src/gems_craft/study/validation.py b/src/gems_craft/study/validation.py new file mode 100644 index 00000000..1539c0f5 --- /dev/null +++ b/src/gems_craft/study/validation.py @@ -0,0 +1,63 @@ +# Copyright (c) 2024, RTE (https://www.rte-france.com) +# +# See AUTHORS.txt +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# SPDX-License-Identifier: MPL-2.0 +# +# This file is part of the Antares project. + +"""Cross-validation of a resolved system against the models and data it refers to. + +Kept apart from `resolve_components.py` (which only resolves the parsed system +into its runtime objects) and from `study.py` (which only holds them together) +— mirroring `optim_config/parsing.py` and `optim_config/validation.py`. +""" + +from typing import Dict + +from gems_craft.model import Model +from gems_craft.study.study import Study +from gems_craft.study.system import System + + +def check_component_models(system: System, input_models: Dict[str, Model]) -> bool: + """ + Checks if all components in the System have a valid model from the library. + Returns True if all components are consistent, raises ValueError otherwise. + """ + # TODO: Update this check to verify that each component has a valid model from the lib it refers to (and not all libs) + model_ids_set = input_models.keys() + for component in system.all_components: + if component.model.id not in model_ids_set: + raise ValueError( + f"Error: Component {component.id} has invalid model ID: {component.model.id}" + ) + return True + + +def check_data_requirements(study: Study) -> None: + """Validate that the database supplies data for every parameter of every + component defined in the system. + + Raises + ------ + ValueError + If a required data entry is missing or its time/scenario structure + does not match what the model parameter expects. + """ + for component in study.system.components: + for param in component.model.parameters.values(): + data_structure = study.database.get_data(component.id, param.name) + + if not data_structure.check_requirement( + component.model.parameters[param.name].structure.time, + component.model.parameters[param.name].structure.scenario, + ): + raise ValueError( + f"Data inconsistency for component: {component.id}, " + f"parameter: {param.name}. Requirement not met." + ) diff --git a/src/gems_runner/simulation/optimization.py b/src/gems_runner/simulation/optimization.py index 71752c58..30706219 100644 --- a/src/gems_runner/simulation/optimization.py +++ b/src/gems_runner/simulation/optimization.py @@ -48,6 +48,7 @@ from gems_craft.study.parsing import IntegerStrategyId from gems_craft.study.study import Study from gems_craft.study.system import Component +from gems_craft.study.validation import check_data_requirements from gems_runner.simulation.linearize import ( VectorizedExpr, VectorizedLinearExprBuilder, @@ -1071,7 +1072,7 @@ def build_problem( problem is built. Entries whose variable is time-independent, or is absent from this block, are ignored. """ - study.check_consistency() + check_data_requirements(study) oob_filter = OutOfBoundsFilter(optim_config) if optim_config is not None else None builder = _OptimizationProblemBuilder( @@ -1145,7 +1146,7 @@ def build_decomposed_problems( """ from gems_craft.optim_config.parsing import ElementLocation - study.check_consistency() + check_data_requirements(study) oob_filter = OutOfBoundsFilter(optim_config) diff --git a/tests/e2e/functional/perf_pypsa.py b/tests/e2e/functional/perf_pypsa.py index 50943679..e5f26433 100644 --- a/tests/e2e/functional/perf_pypsa.py +++ b/tests/e2e/functional/perf_pypsa.py @@ -12,10 +12,10 @@ from gems_craft.study.parsing import parse_yaml_system from gems_craft.study.resolve_components import ( build_data_base, - consistency_check, resolve_system, ) from gems_craft.study.system import System +from gems_craft.study.validation import check_component_models from gems_runner.simulation import TimeBlock, build_problem @@ -30,7 +30,7 @@ def setup_data(pypsa_dir: Path) -> Tuple[System, DataBase]: input_study = parse_yaml_system(c) lib_dict = resolve_library([input_library]) system = resolve_system(input_study, lib_dict) - consistency_check(system, lib_dict["pypsa_models"].models) + check_component_models(system, lib_dict["pypsa_models"].models) database = build_data_base(input_study, series_dir) return system, database diff --git a/tests/e2e/functional/test_component_dependent_time_shift.py b/tests/e2e/functional/test_component_dependent_time_shift.py index f9bd5aaa..983e1115 100644 --- a/tests/e2e/functional/test_component_dependent_time_shift.py +++ b/tests/e2e/functional/test_component_dependent_time_shift.py @@ -100,9 +100,9 @@ from gems_craft.study.parsing import parse_yaml_system from gems_craft.study.resolve_components import ( build_data_base, - consistency_check, resolve_system, ) +from gems_craft.study.validation import check_component_models from gems_runner.simulation import TimeBlock, build_problem from tests.e2e.functional.libs.standard import ( BALANCE_PORT_TYPE, @@ -413,7 +413,7 @@ def test_two_components_different_lags_yaml( lib_dict = resolve_library([input_library]) system = resolve_system(input_system, lib_dict) - consistency_check(system, lib_dict["time_shift_test"].models) + check_component_models(system, lib_dict["time_shift_test"].models) database = build_data_base(input_system, timeseries_dir=_series_dir) diff --git a/tests/e2e/functional/test_libs_yaml_system_yaml.py b/tests/e2e/functional/test_libs_yaml_system_yaml.py index 1e886d2a..99c130d3 100644 --- a/tests/e2e/functional/test_libs_yaml_system_yaml.py +++ b/tests/e2e/functional/test_libs_yaml_system_yaml.py @@ -47,11 +47,11 @@ from gems_craft.study.parsing import SystemSchema, parse_yaml_system from gems_craft.study.resolve_components import ( build_data_base, - consistency_check, resolve_system, ) from gems_craft.study.study import Study from gems_craft.study.system import System +from gems_craft.study.validation import check_component_models from gems_runner.simulation import TimeBlock, build_problem @@ -60,7 +60,7 @@ def test_basic_balance_using_yaml( ) -> None: result_lib = resolve_library([input_library]) system = resolve_system(input_system, result_lib) - consistency_check(system, result_lib["basic"].models) + check_component_models(system, result_lib["basic"].models) database = build_data_base(input_system, None) @@ -87,7 +87,7 @@ def _setup_test(study_file_name: str) -> Study: input_system = parse_yaml_system(c) lib_dict = resolve_library([input_library]) system = resolve_system(input_system, lib_dict) - consistency_check(system, lib_dict["basic"].models) + check_component_models(system, lib_dict["basic"].models) database = build_data_base(input_system, series_dir) return Study(system, database) diff --git a/tests/e2e/functional/test_scenario_builder.py b/tests/e2e/functional/test_scenario_builder.py index c3758171..f46f884f 100644 --- a/tests/e2e/functional/test_scenario_builder.py +++ b/tests/e2e/functional/test_scenario_builder.py @@ -21,10 +21,10 @@ from gems_craft.study.parsing import parse_yaml_system from gems_craft.study.resolve_components import ( build_data_base, - consistency_check, resolve_system, ) from gems_craft.study.scenario_builder import ScenarioBuilder +from gems_craft.study.validation import check_component_models from gems_runner.simulation import build_problem from gems_runner.simulation.time_block import TimeBlock @@ -58,7 +58,7 @@ def test_system_with_scenarization( yaml_comp = parse_yaml_system(file) components = resolve_system(yaml_comp, lib_dict) - consistency_check(components, lib_dict["basic"].models) + check_component_models(components, lib_dict["basic"].models) timeblock = TimeBlock(1, list(range(2))) problem = build_problem(Study(components, database), timeblock, list(range(3))) diff --git a/tests/e2e/functional/test_study_from_folder.py b/tests/e2e/functional/test_study_from_folder.py index f28cc2f8..d742ad14 100644 --- a/tests/e2e/functional/test_study_from_folder.py +++ b/tests/e2e/functional/test_study_from_folder.py @@ -2,10 +2,13 @@ from pathlib import Path import pandas as pd +import pytest from gems_craft.study.folder import load_study from gems_runner.study.runner import run_study +TAXONOMY_FILE = Path("input") / "taxonomy.yml" + def test_load_study(): study_dir = Path(__file__).parent / "studies" / "7_4" @@ -27,3 +30,44 @@ def test_run_study(tmp_path: Path) -> None: assert len(output_files) == 1 df = pd.read_csv(output_files[0]) assert "objective-value" in df["output"].values + + +def _study_declaring_taxonomy(tmp_path: Path) -> Path: + """Copy the 7_4 study, making one of its libraries declare a taxonomy.""" + study_dir = tmp_path / "7_4" + shutil.copytree(Path(__file__).parent / "studies" / "7_4", study_dir) + + lib_path = study_dir / "input" / "model-libraries" / "antares_historic.yml" + lib_path.write_text( + lib_path.read_text().replace( + " id: antares-historic-weo", + " id: antares-historic-weo\n taxonomy: study_taxonomy", + 1, + ) + ) + return study_dir + + +def test_load_study_checks_libraries_against_study_taxonomy(tmp_path: Path) -> None: + study_dir = _study_declaring_taxonomy(tmp_path) + (study_dir / TAXONOMY_FILE).write_text( + "taxonomy:\n id: study_taxonomy\n categories:\n - id: production\n" + ) + + study = load_study(study_dir) # must not raise + assert len(study.system.components) == 12 + + +def test_load_study_raises_when_taxonomy_file_is_missing(tmp_path: Path) -> None: + study_dir = _study_declaring_taxonomy(tmp_path) + + with pytest.raises(ValueError, match="no taxonomy was provided"): + load_study(study_dir) + + +def test_load_study_raises_on_taxonomy_id_mismatch(tmp_path: Path) -> None: + study_dir = _study_declaring_taxonomy(tmp_path) + (study_dir / TAXONOMY_FILE).write_text("taxonomy:\n id: other_taxonomy\n") + + with pytest.raises(ValueError, match="other_taxonomy"): + load_study(study_dir) diff --git a/tests/unittests/gems_craft/lib_parsing/test_taxonomy_check.py b/tests/unittests/gems_craft/lib_parsing/test_taxonomy_check.py index b573476f..44fa3fff 100644 --- a/tests/unittests/gems_craft/lib_parsing/test_taxonomy_check.py +++ b/tests/unittests/gems_craft/lib_parsing/test_taxonomy_check.py @@ -20,9 +20,12 @@ Taxonomy, TaxonomyCategory, TaxonomyItem, - check_library_against_taxonomy, load_taxonomy, ) +from gems_craft.model.validation import ( + check_library_against_taxonomy, + validate_libraries_against_taxonomy, +) def _make_taxonomy(*categories: TaxonomyCategory) -> Taxonomy: @@ -418,3 +421,81 @@ def test_model_exposing_all_required_fields_is_valid() -> None: - id: technology """) check_library_against_taxonomy(lib, taxonomy) # must not raise + + +# --- validate_libraries_against_taxonomy --- + +_CONFORMING_LIB = """ +library: + id: mylib + taxonomy: test_taxonomy + port-types: + - id: flow + fields: + - id: flow + models: + - id: generator + taxonomy-category: production + ports: + - id: injection_port + type: flow +""" + +_VIOLATING_LIB = """ +library: + id: mylib + taxonomy: test_taxonomy + models: + - id: generator + taxonomy-category: production +""" + +# Same violating model, but the library itself does not opt in to a taxonomy. +_LIB_WITHOUT_TAXONOMY = """ +library: + id: mylib + models: + - id: generator + taxonomy-category: production +""" + + +def test_library_declaring_taxonomy_is_checked() -> None: + taxonomy = _make_taxonomy(_make_category("production", ["injection_port"])) + lib = _parse_lib(_CONFORMING_LIB) + validate_libraries_against_taxonomy([lib], taxonomy) # must not raise + assert lib.taxonomy == "test_taxonomy" + + +def test_library_declaring_taxonomy_raises_on_violation() -> None: + taxonomy = _make_taxonomy(_make_category("production", ["injection_port"])) + with pytest.raises(ValueError, match="injection_port"): + validate_libraries_against_taxonomy([_parse_lib(_VIOLATING_LIB)], taxonomy) + + +def test_library_declaring_taxonomy_without_argument_raises() -> None: + with pytest.raises(ValueError, match="no taxonomy was provided"): + validate_libraries_against_taxonomy([_parse_lib(_CONFORMING_LIB)], None) + + +def test_library_with_mismatched_taxonomy_id_raises() -> None: + taxonomy = Taxonomy( + id="other_taxonomy", + categories=[_make_category("production", ["injection_port"])], + ) + with pytest.raises(ValueError, match="other_taxonomy"): + validate_libraries_against_taxonomy([_parse_lib(_CONFORMING_LIB)], taxonomy) + + +def test_library_without_declared_taxonomy_is_not_checked() -> None: + """A model may carry a taxonomy-category without the library opting in.""" + taxonomy = _make_taxonomy(_make_category("production", ["injection_port"])) + lib = _parse_lib(_LIB_WITHOUT_TAXONOMY) + validate_libraries_against_taxonomy([lib], taxonomy) # must not raise + assert lib.taxonomy is None + + +def test_parsing_a_declaring_library_does_not_validate_it() -> None: + """Parsing is pure reading: conformance is the caller's business.""" + lib = _parse_lib(_VIOLATING_LIB) # must not raise + assert lib.taxonomy == "test_taxonomy" diff --git a/tests/unittests/gems_craft/system/test_data_consistency.py b/tests/unittests/gems_craft/system/test_data_consistency.py index 400a1990..dcab18b1 100644 --- a/tests/unittests/gems_craft/system/test_data_consistency.py +++ b/tests/unittests/gems_craft/system/test_data_consistency.py @@ -41,6 +41,7 @@ create_component, ) from gems_craft.study.data import load_ts_from_file +from gems_craft.study.validation import check_data_requirements from tests.unittests.gems_craft.system.libs.standard import ( BALANCE_PORT_TYPE, CONSTANT, @@ -152,7 +153,7 @@ def test_requirements_consistency_demand_model_fix_ok( # When # No ValueError should be raised - Study(mock_network, database).check_consistency() + check_data_requirements(Study(mock_network, database)) def test_requirements_consistency_generator_model_ok(mock_network: System) -> None: @@ -165,7 +166,7 @@ def test_requirements_consistency_generator_model_ok(mock_network: System) -> No database.add_data("D", "demand", ConstantData(30)) # When - Study(mock_network, database).check_consistency() + check_data_requirements(Study(mock_network, database)) def test_consistency_generation_time_free_for_constant_model_raises_exception( @@ -184,7 +185,7 @@ def test_consistency_generation_time_free_for_constant_model_raises_exception( # When with pytest.raises(ValueError, match="Data inconsistency"): - Study(mock_network, database).check_consistency() + check_data_requirements(Study(mock_network, database)) def test_requirements_consistency_demand_model_time_varying_ok( @@ -200,7 +201,7 @@ def test_requirements_consistency_demand_model_time_varying_ok( # When # No ValueError should be raised - Study(mock_network, database).check_consistency() + check_data_requirements(Study(mock_network, database)) def test_requirements_consistency_time_varying_parameter_with_correct_data_passes( @@ -225,7 +226,7 @@ def test_requirements_consistency_time_varying_parameter_with_correct_data_passe system.connect(PortRef(gen, "balance_port"), PortRef(node, "balance_port")) # No ValueError should be raised - Study(system, database).check_consistency() + check_data_requirements(Study(system, database)) @pytest.mark.parametrize( @@ -270,7 +271,7 @@ def test_requirements_consistency_time_varying_parameter_with_scenario_varying_d # When # ValueError should be raised with pytest.raises(ValueError, match="Data inconsistency"): - Study(system, database).check_consistency() + check_data_requirements(Study(system, database)) @pytest.mark.parametrize( @@ -306,7 +307,7 @@ def test_requirements_consistency_scenario_varying_parameter_with_time_varying_d # ValueError should be raised with pytest.raises(ValueError, match="Data inconsistency"): - Study(system, database).check_consistency() + check_data_requirements(Study(system, database)) def test_requirements_consistency_scenario_varying_parameter_with_correct_data_passes( @@ -331,7 +332,7 @@ def test_requirements_consistency_scenario_varying_parameter_with_correct_data_p system.add_component(gen) # No ValueError should be raised - Study(system, database).check_consistency() + check_data_requirements(Study(system, database)) def test_load_data_from_txt() -> None: diff --git a/tests/unittests/gems_craft/system_parsing/test_components_parsing.py b/tests/unittests/gems_craft/system_parsing/test_components_parsing.py index 7173a3fe..00a3978d 100644 --- a/tests/unittests/gems_craft/system_parsing/test_components_parsing.py +++ b/tests/unittests/gems_craft/system_parsing/test_components_parsing.py @@ -18,7 +18,8 @@ load_input_system, parse_yaml_system, ) -from gems_craft.study.resolve_components import consistency_check, resolve_system +from gems_craft.study.resolve_components import resolve_system +from gems_craft.study.validation import check_component_models COMPO_FILE = Path(__file__).parent / "systems/system.yml" @@ -50,12 +51,12 @@ def test_parsing_components_ok( assert len(result.connections) == 2 -def test_consistency_check_ok( +def test_check_component_models_ok( input_system: SystemSchema, input_library: LibrarySchema ) -> None: result_lib = resolve_library([input_library]) result_system = resolve_system(input_system, result_lib) - consistency_check(result_system, result_lib["basic"].models) + check_component_models(result_system, result_lib["basic"].models) def test_load_input_system_ok(tmp_path: Path) -> None: @@ -93,7 +94,7 @@ def test_load_input_system_missing_file_raises_error() -> None: load_input_system(missing) -def test_consistency_check_ko( +def test_check_component_models_ko( input_system: SystemSchema, input_library: LibrarySchema ) -> None: result_lib = resolve_library([input_library]) @@ -103,7 +104,7 @@ def test_consistency_check_ko( ValueError, match=r"Error: Component G has invalid model ID: basic.generator", ): - consistency_check(result_comp, result_lib["basic"].models) + check_component_models(result_comp, result_lib["basic"].models) # ---------------------------------------------------------------------------