From 0c80b26ebd45a915702910723f6caaf41a45dbec Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 10:11:02 +0000 Subject: [PATCH 1/2] refactor(taxonomy): split library validation out of parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses tbittar's review on #278: cross-artifact validation had been put inside parse_yaml_library, which is the reading/validating mixing that #265 set out to undo. - New gems_craft/model/validation.py holds check_library_against_taxonomy (moved unchanged from taxonomy.py, with its _missing helper) plus validate_libraries_against_taxonomy, which absorbs the declared-id and missing-taxonomy checks. - parse_yaml_library goes back to a pure reader: no taxonomy argument, no validation. input_libs likewise loses the argument. - load_study reads input/taxonomy.yml and calls the validation explicitly after parsing, alongside consistency_check. - taxonomy.py no longer imports parsing, so the TYPE_CHECKING guard added to break that cycle is gone — the cycle no longer exists. - Acts on the TODO left by #265: consistency_check moves from resolve_components.py to a new gems_craft/study/validation.py; its callers are repointed and a now-dead Model import is dropped. Conformance is enforced when a study is loaded rather than on every parse, matching how consistency_check and validate_optim_config already work. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DGFvLY7afUMBnwfHgKHHAD --- AGENTS.md | 2 +- docs/CHANGELOG.md | 17 ++- src/gems_craft/model/parsing.py | 24 +--- src/gems_craft/model/taxonomy.py | 91 +----------- src/gems_craft/model/validation.py | 131 ++++++++++++++++++ src/gems_craft/study/folder.py | 6 +- src/gems_craft/study/resolve_components.py | 18 --- src/gems_craft/study/study.py | 4 +- src/gems_craft/study/validation.py | 38 +++++ src/gems_runner/main/main.py | 7 +- tests/e2e/functional/perf_pypsa.py | 2 +- .../test_component_dependent_time_shift.py | 2 +- .../functional/test_libs_yaml_system_yaml.py | 2 +- tests/e2e/functional/test_scenario_builder.py | 2 +- .../lib_parsing/test_taxonomy_check.py | 40 ++++-- .../system_parsing/test_components_parsing.py | 3 +- 16 files changed, 224 insertions(+), 165 deletions(-) create mode 100644 src/gems_craft/model/validation.py create mode 100644 src/gems_craft/study/validation.py diff --git a/AGENTS.md b/AGENTS.md index 600989d0..eef978bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,7 +72,7 @@ The codebase is split into three packages along a solver-dependency boundary: **`gems_craft/model/`** — Immutable model templates. - `Model`: defines component behavior (parameters, variables, constraints, ports) - `Library`: a collection of models, loaded from YAML -- `Taxonomy` (`taxonomy.py`): categories naming the items a model must expose. Models opt in via `taxonomy-category`; `check_library_against_taxonomy` enforces conformance, called from `parse_yaml_library` for libraries declaring a `taxonomy`. The caller supplies the `Taxonomy`; `load_study` reads it from the optional `input/taxonomy.yml`. +- `Taxonomy` (`taxonomy.py`): categories naming the items a model must expose. Models opt in via `taxonomy-category`. `taxonomy.py` holds the data and `load_taxonomy` only; conformance lives in `validation.py` (`check_library_against_taxonomy`, and `validate_libraries_against_taxonomy` for libraries declaring a `taxonomy`), which `load_study` calls after parsing — `parse_yaml_library` stays a pure reader. `load_study` reads the `Taxonomy` from the optional `input/taxonomy.yml`. - `PortTypeSchema` (`parsing.py`) also declares the hybrid-only `area-connection` (`AreaConnectionSchema`) and `thermal-capacity-connection` (`PortThermalCapacitySchema`) fields; `resolve_library.py`'s `_convert_port_type` parses and discards them for every library, hybrid or not. **`gems_craft/expression/`** — Mathematical expression language and AST (structural/static analysis only — no numeric evaluation). diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 363c7b59..dfe1f081 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,9 +9,15 @@ All notable changes to GemsPy are documented here. *N+1*'s first timestep to block *N*'s **last** timestep — a different absolute timestep. - **linopy upgraded to `>=0.9.0`** - the minimum supported Python version rises to **3.11** accordingly (linopy 0.9 requires Python >= 3.11). -- **Breaking** - parsing a library that declares a `taxonomy` raises `ValueError` - if no taxonomy is supplied, or if its id differs from the declared one. +- **Breaking** - loading a study whose library declares a `taxonomy` raises + `ValueError` if no taxonomy is supplied, or if its id differs from the declared + one. - **Breaking** - `TaxonomyData` renamed to `TaxonomySchema`; no alias kept. +- **Breaking** - `check_library_against_taxonomy` moved from + `gems_craft.model.taxonomy` to the new `gems_craft.model.validation`, and + `consistency_check` from `gems_craft.study.resolve_components` to the new + `gems_craft.study.validation`. Reading and validating are now separate modules, + as in `optim_config/`. No behavior change; import paths only. ### Added - **Integer strategy and thermal heuristics** - components can now set @@ -30,9 +36,10 @@ All notable changes to GemsPy are documented here. model-build time; using them inside constraints, binding-constraints, objective contributions, or variable bounds raises a `ValueError`. -- **Taxonomy conformance checked at parse time** - `parse_yaml_library` takes an - optional `taxonomy` and checks every library declaring a `taxonomy` field. - `load_study` reads it from the optional `input/taxonomy.yml`. +- **Taxonomy conformance checked when a study is loaded** - `load_study` reads + the optional `input/taxonomy.yml` and calls + `validate_libraries_against_taxonomy` on every library declaring a `taxonomy` + field. `parse_yaml_library` is unchanged and performs no validation. ### Fixed - **Standard library parsing now accepts hybrid port-type fields** - diff --git a/src/gems_craft/model/parsing.py b/src/gems_craft/model/parsing.py index bbacbed5..cadb383e 100644 --- a/src/gems_craft/model/parsing.py +++ b/src/gems_craft/model/parsing.py @@ -17,7 +17,6 @@ from pydantic import ConfigDict, Field, ValidationError from yaml import safe_dump, safe_load -from gems_craft.model.taxonomy import Taxonomy, check_library_against_taxonomy from gems_craft.utils import ModifiedBaseModel @@ -124,31 +123,12 @@ class LibrarySchema(ModifiedBaseModel): version: Optional[str] = None -def _check_declared_taxonomy( - library: LibrarySchema, taxonomy: Optional[Taxonomy] -) -> None: - """Check a library against the taxonomy it declares conformance to.""" - declared = f"Library '{library.id}' declares taxonomy '{library.taxonomy}'" - if taxonomy is None: - raise ValueError( - f"{declared} but no taxonomy was provided to check it against." - ) - if library.taxonomy != taxonomy.id: - raise ValueError(f"{declared} but was checked against '{taxonomy.id}'.") - check_library_against_taxonomy(library, taxonomy) - - -def parse_yaml_library( - input: typing.TextIO, taxonomy: Optional[Taxonomy] = None -) -> LibrarySchema: +def parse_yaml_library(input: typing.TextIO) -> LibrarySchema: tree = safe_load(input) try: - library = LibrarySchema.model_validate(tree["library"]) + return LibrarySchema.model_validate(tree["library"]) except ValidationError as e: raise ValueError(f"An error occurred during parsing: {e}") - if library.taxonomy is not None: - _check_declared_taxonomy(library, taxonomy) - return library def write_yaml_library(library: LibrarySchema, path: Path) -> None: diff --git a/src/gems_craft/model/taxonomy.py b/src/gems_craft/model/taxonomy.py index 6eba7e18..b2eb2410 100644 --- a/src/gems_craft/model/taxonomy.py +++ b/src/gems_craft/model/taxonomy.py @@ -12,17 +12,13 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Callable, Dict, List, Optional +from typing import List, Optional import yaml from pydantic import Field from gems_craft.utils import ModifiedBaseModel -if TYPE_CHECKING: - # Annotations only — parsing.py imports this module, so this would be circular. - from gems_craft.model.parsing import LibrarySchema, ModelSchema - class TaxonomyItem(ModifiedBaseModel): id: str @@ -63,88 +59,3 @@ def load_taxonomy(taxonomy_file: Path) -> Taxonomy: return Taxonomy( id=data.id, description=data.description, categories=data.categories ) - - -def _missing( - required: List[TaxonomyItem], exposed: List, exposed_key: Callable -) -> List[str]: - """Return the sorted taxonomy item ids not exposed by the model. - - Taxonomy items are always identified by their ``id``; ``exposed_key`` maps each - model-side item to the identifier to compare against (e.g. the ``port.field`` - string for port-field-definitions). - """ - return sorted( - {item.id for item in required} - {exposed_key(item) for item in exposed} - ) - - -def check_library_against_taxonomy( - library: "LibrarySchema", taxonomy: Taxonomy -) -> None: - """ - Validates that every model declaring a taxonomy_category: - 1. References a category that exists in the taxonomy. - 2. Exposes all variables, parameters, ports, port-field-definitions, - constraints, binding-constraints, extra-outputs and properties listed - in that taxonomy category. - - Raises ValueError describing the first violation found. - """ - categories: Dict[str, TaxonomyCategory] = {c.id: c for c in taxonomy.categories} - - by_id: Callable = lambda x: x.id - - # Each entry maps a human-readable field-group name to the required items - # (from the taxonomy category) and the items exposed by the model, plus the - # function identifying a model-side item within that group. Taxonomy items are - # homogeneous (``TaxonomyItem``) and always identified by their ``id``. - def field_groups( - category: TaxonomyCategory, model_schema: "ModelSchema" - ) -> List[tuple]: - port_field_key: Callable = lambda d: f"{d.port}.{d.field}" - return [ - ("variable", category.variables, model_schema.variables, by_id), - ("parameter", category.parameters, model_schema.parameters, by_id), - ("port", category.ports, model_schema.ports, by_id), - ( - "port-field-definition", - category.port_field_definitions, - model_schema.port_field_definitions, - port_field_key, - ), - ("constraint", category.constraints, model_schema.constraints, by_id), - ( - "binding-constraint", - category.binding_constraints, - model_schema.binding_constraints, - by_id, - ), - ( - "extra-output", - category.extra_outputs, - model_schema.extra_outputs or [], - by_id, - ), - ("property", category.properties, model_schema.properties, by_id), - ] - - for model_schema in library.models: - cat_id = model_schema.taxonomy_category - if cat_id is None: - continue - - if cat_id not in categories: - raise ValueError( - f"Model '{model_schema.id}' references taxonomy category '{cat_id}' " - f"which does not exist in taxonomy '{taxonomy.id}'." - ) - - category = categories[cat_id] - for group_name, required, exposed, key in field_groups(category, model_schema): - missing = _missing(required, exposed, key) - if missing: - raise ValueError( - f"Model '{model_schema.id}' (taxonomy-category: '{cat_id}') is " - f"missing {group_name}(s) required by the taxonomy: {missing}." - ) diff --git a/src/gems_craft/model/validation.py b/src/gems_craft/model/validation.py new file mode 100644 index 00000000..6a6ee951 --- /dev/null +++ b/src/gems_craft/model/validation.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026, RTE (https://www.rte-france.com) +# +# See AUTHORS.txt +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# SPDX-License-Identifier: MPL-2.0 +# +# This file is part of the Antares project. + +"""Cross-validation of parsed model libraries. + +Kept apart from `parsing.py` (which only reads YAML into schemas) and from +`taxonomy.py` (which only holds the taxonomy data and reads it from disk), so +that reading and validating stay separate concerns — mirroring +`optim_config/parsing.py` and `optim_config/validation.py`. +""" + +from typing import Callable, Dict, List, Optional + +from gems_craft.model.parsing import LibrarySchema, ModelSchema +from gems_craft.model.taxonomy import Taxonomy, TaxonomyCategory, TaxonomyItem + + +def validate_libraries_against_taxonomy( + libraries: List[LibrarySchema], taxonomy: Optional[Taxonomy] +) -> None: + """Check every library declaring a ``taxonomy`` field against ``taxonomy``. + + Libraries that do not declare a taxonomy are left alone, even when some of + their models carry a ``taxonomy-category``. + + Raises ValueError if a declaring library has no taxonomy to check against, + if it declares a different taxonomy id, or if any of its models violates it. + """ + for library in libraries: + if library.taxonomy is None: + continue + declared = f"Library '{library.id}' declares taxonomy '{library.taxonomy}'" + if taxonomy is None: + raise ValueError( + f"{declared} but no taxonomy was provided to check it against." + ) + if library.taxonomy != taxonomy.id: + raise ValueError(f"{declared} but was checked against '{taxonomy.id}'.") + check_library_against_taxonomy(library, taxonomy) + + +def _missing( + required: List[TaxonomyItem], exposed: List, exposed_key: Callable +) -> List[str]: + """Return the sorted taxonomy item ids not exposed by the model. + + Taxonomy items are always identified by their ``id``; ``exposed_key`` maps each + model-side item to the identifier to compare against (e.g. the ``port.field`` + string for port-field-definitions). + """ + return sorted( + {item.id for item in required} - {exposed_key(item) for item in exposed} + ) + + +def check_library_against_taxonomy(library: LibrarySchema, taxonomy: Taxonomy) -> None: + """ + Validates that every model declaring a taxonomy_category: + 1. References a category that exists in the taxonomy. + 2. Exposes all variables, parameters, ports, port-field-definitions, + constraints, binding-constraints, extra-outputs and properties listed + in that taxonomy category. + + Raises ValueError describing the first violation found. + """ + categories: Dict[str, TaxonomyCategory] = {c.id: c for c in taxonomy.categories} + + by_id: Callable = lambda x: x.id + + # Each entry maps a human-readable field-group name to the required items + # (from the taxonomy category) and the items exposed by the model, plus the + # function identifying a model-side item within that group. Taxonomy items are + # homogeneous (``TaxonomyItem``) and always identified by their ``id``. + def field_groups( + category: TaxonomyCategory, model_schema: ModelSchema + ) -> List[tuple]: + port_field_key: Callable = lambda d: f"{d.port}.{d.field}" + return [ + ("variable", category.variables, model_schema.variables, by_id), + ("parameter", category.parameters, model_schema.parameters, by_id), + ("port", category.ports, model_schema.ports, by_id), + ( + "port-field-definition", + category.port_field_definitions, + model_schema.port_field_definitions, + port_field_key, + ), + ("constraint", category.constraints, model_schema.constraints, by_id), + ( + "binding-constraint", + category.binding_constraints, + model_schema.binding_constraints, + by_id, + ), + ( + "extra-output", + category.extra_outputs, + model_schema.extra_outputs or [], + by_id, + ), + ("property", category.properties, model_schema.properties, by_id), + ] + + for model_schema in library.models: + cat_id = model_schema.taxonomy_category + if cat_id is None: + continue + + if cat_id not in categories: + raise ValueError( + f"Model '{model_schema.id}' references taxonomy category '{cat_id}' " + f"which does not exist in taxonomy '{taxonomy.id}'." + ) + + category = categories[cat_id] + for group_name, required, exposed, key in field_groups(category, model_schema): + missing = _missing(required, exposed, key) + if missing: + raise ValueError( + f"Model '{model_schema.id}' (taxonomy-category: '{cat_id}') is " + f"missing {group_name}(s) required by the taxonomy: {missing}." + ) diff --git a/src/gems_craft/study/folder.py b/src/gems_craft/study/folder.py index 79020138..838b1bba 100644 --- a/src/gems_craft/study/folder.py +++ b/src/gems_craft/study/folder.py @@ -15,14 +15,15 @@ from gems_craft.model.parsing import parse_yaml_library from gems_craft.model.resolve_library import resolve_library from gems_craft.model.taxonomy import load_taxonomy +from gems_craft.model.validation import validate_libraries_against_taxonomy from gems_craft.study.parsing import parse_yaml_system from gems_craft.study.resolve_components import ( build_data_base, - consistency_check, resolve_system, ) from gems_craft.study.scenario_builder import ScenarioBuilder from gems_craft.study.study import Study +from gems_craft.study.validation import consistency_check def load_study(study_dir: Path) -> Study: @@ -50,7 +51,8 @@ def load_study(study_dir: Path) -> Study: input_libraries = [] for lib_file in lib_folder.glob("*.yml"): with lib_file.open() as lib: - input_libraries.append(parse_yaml_library(lib, taxonomy=taxonomy)) + input_libraries.append(parse_yaml_library(lib)) + validate_libraries_against_taxonomy(input_libraries, taxonomy) with system_file.open() as c: input_study = parse_yaml_system(c) diff --git a/src/gems_craft/study/resolve_components.py b/src/gems_craft/study/resolve_components.py index ced3457b..83197607 100644 --- a/src/gems_craft/study/resolve_components.py +++ b/src/gems_craft/study/resolve_components.py @@ -12,7 +12,6 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple, Union -from gems_craft.model import Model from gems_craft.model.library import Library from gems_craft.study import ( Component, @@ -132,23 +131,6 @@ def _get_component_by_id( return components_dict.get(component_id) -# TODO: cross-validation logic mixed into a "resolve" (preprocessing) module — -# move to a dedicated gems_craft/study/validation.py, mirroring optim_config/. -def consistency_check(system: System, input_models: Dict[str, Model]) -> bool: - """ - Checks if all components in the System have a valid model from the library. - Returns True if all components are consistent, raises ValueError otherwise. - """ - # TODO: Update this consistency check to check if each component have a valid model from the lib it refers to (and not all libs) - model_ids_set = input_models.keys() - for component in system.all_components: - if component.model.id not in model_ids_set: - raise ValueError( - f"Error: Component {component.id} has invalid model ID: {component.model.id}" - ) - return True - - def build_data_base( input_system: SystemSchema, timeseries_dir: Optional[Path], diff --git a/src/gems_craft/study/study.py b/src/gems_craft/study/study.py index 32080be0..4c63db00 100644 --- a/src/gems_craft/study/study.py +++ b/src/gems_craft/study/study.py @@ -54,8 +54,8 @@ def models(self) -> Dict[str, Model]: } # TODO: this is a second, disjoint consistency check alongside - # resolve_components.consistency_check() — consider consolidating both - # into one validation module for System/Study. + # study/validation.py's consistency_check() — consider consolidating both + # into that validation module. def check_consistency(self) -> None: """Validate that the database supplies data for every parameter of every component defined in the system. diff --git a/src/gems_craft/study/validation.py b/src/gems_craft/study/validation.py new file mode 100644 index 00000000..924a1486 --- /dev/null +++ b/src/gems_craft/study/validation.py @@ -0,0 +1,38 @@ +# Copyright (c) 2024, RTE (https://www.rte-france.com) +# +# See AUTHORS.txt +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# SPDX-License-Identifier: MPL-2.0 +# +# This file is part of the Antares project. + +"""Cross-validation of a resolved system against the models it refers to. + +Kept apart from `resolve_components.py`, which only resolves the parsed system +into its runtime objects — mirroring `optim_config/parsing.py` and +`optim_config/validation.py`. +""" + +from typing import Dict + +from gems_craft.model import Model +from gems_craft.study.system import System + + +def consistency_check(system: System, input_models: Dict[str, Model]) -> bool: + """ + Checks if all components in the System have a valid model from the library. + Returns True if all components are consistent, raises ValueError otherwise. + """ + # TODO: Update this consistency check to check if each component have a valid model from the lib it refers to (and not all libs) + model_ids_set = input_models.keys() + for component in system.all_components: + if component.model.id not in model_ids_set: + raise ValueError( + f"Error: Component {component.id} has invalid model ID: {component.model.id}" + ) + return True diff --git a/src/gems_runner/main/main.py b/src/gems_runner/main/main.py index 93c6be3e..a0d33721 100644 --- a/src/gems_runner/main/main.py +++ b/src/gems_runner/main/main.py @@ -16,7 +16,6 @@ from gems_craft.model.library import Library from gems_craft.model.parsing import parse_yaml_library from gems_craft.model.resolve_library import resolve_library -from gems_craft.model.taxonomy import Taxonomy from gems_craft.optim_config.parsing import OptimConfig from gems_craft.study import Study from gems_craft.study.data import DataBase @@ -27,14 +26,12 @@ from gems_runner.study.runner import run_study -def input_libs( - yaml_lib_paths: List[Path], taxonomy: Optional[Taxonomy] = None -) -> Dict[str, Library]: +def input_libs(yaml_lib_paths: List[Path]) -> Dict[str, Library]: yaml_libraries = [] yaml_library_ids = set() for path in yaml_lib_paths: with path.open("r") as file: - yaml_lib = parse_yaml_library(file, taxonomy=taxonomy) + yaml_lib = parse_yaml_library(file) if yaml_lib.id in yaml_library_ids: raise ValueError(f"The identifier '{yaml_lib.id}' is defined twice") yaml_libraries.append(yaml_lib) diff --git a/tests/e2e/functional/perf_pypsa.py b/tests/e2e/functional/perf_pypsa.py index 50943679..6fd91fe3 100644 --- a/tests/e2e/functional/perf_pypsa.py +++ b/tests/e2e/functional/perf_pypsa.py @@ -12,10 +12,10 @@ from gems_craft.study.parsing import parse_yaml_system from gems_craft.study.resolve_components import ( build_data_base, - consistency_check, resolve_system, ) from gems_craft.study.system import System +from gems_craft.study.validation import consistency_check from gems_runner.simulation import TimeBlock, build_problem diff --git a/tests/e2e/functional/test_component_dependent_time_shift.py b/tests/e2e/functional/test_component_dependent_time_shift.py index f9bd5aaa..070976c5 100644 --- a/tests/e2e/functional/test_component_dependent_time_shift.py +++ b/tests/e2e/functional/test_component_dependent_time_shift.py @@ -100,9 +100,9 @@ from gems_craft.study.parsing import parse_yaml_system from gems_craft.study.resolve_components import ( build_data_base, - consistency_check, resolve_system, ) +from gems_craft.study.validation import consistency_check from gems_runner.simulation import TimeBlock, build_problem from tests.e2e.functional.libs.standard import ( BALANCE_PORT_TYPE, diff --git a/tests/e2e/functional/test_libs_yaml_system_yaml.py b/tests/e2e/functional/test_libs_yaml_system_yaml.py index 1e886d2a..c8d3bad9 100644 --- a/tests/e2e/functional/test_libs_yaml_system_yaml.py +++ b/tests/e2e/functional/test_libs_yaml_system_yaml.py @@ -47,11 +47,11 @@ from gems_craft.study.parsing import SystemSchema, parse_yaml_system from gems_craft.study.resolve_components import ( build_data_base, - consistency_check, resolve_system, ) from gems_craft.study.study import Study from gems_craft.study.system import System +from gems_craft.study.validation import consistency_check from gems_runner.simulation import TimeBlock, build_problem diff --git a/tests/e2e/functional/test_scenario_builder.py b/tests/e2e/functional/test_scenario_builder.py index c3758171..7a20dc1e 100644 --- a/tests/e2e/functional/test_scenario_builder.py +++ b/tests/e2e/functional/test_scenario_builder.py @@ -21,10 +21,10 @@ from gems_craft.study.parsing import parse_yaml_system from gems_craft.study.resolve_components import ( build_data_base, - consistency_check, resolve_system, ) from gems_craft.study.scenario_builder import ScenarioBuilder +from gems_craft.study.validation import consistency_check from gems_runner.simulation import build_problem from gems_runner.simulation.time_block import TimeBlock diff --git a/tests/unittests/gems_craft/lib_parsing/test_taxonomy_check.py b/tests/unittests/gems_craft/lib_parsing/test_taxonomy_check.py index c792f736..44fa3fff 100644 --- a/tests/unittests/gems_craft/lib_parsing/test_taxonomy_check.py +++ b/tests/unittests/gems_craft/lib_parsing/test_taxonomy_check.py @@ -12,7 +12,6 @@ import io from pathlib import Path -from typing import Optional import pytest @@ -21,9 +20,12 @@ Taxonomy, TaxonomyCategory, TaxonomyItem, - check_library_against_taxonomy, load_taxonomy, ) +from gems_craft.model.validation import ( + check_library_against_taxonomy, + validate_libraries_against_taxonomy, +) def _make_taxonomy(*categories: TaxonomyCategory) -> Taxonomy: @@ -34,8 +36,8 @@ def _make_category(cat_id: str, port_ids: list[str]) -> TaxonomyCategory: return TaxonomyCategory(id=cat_id, ports=[TaxonomyItem(id=p) for p in port_ids]) -def _parse_lib(yaml_content: str, taxonomy: Optional[Taxonomy] = None): - return parse_yaml_library(io.StringIO(yaml_content), taxonomy=taxonomy) +def _parse_lib(yaml_content: str): + return parse_yaml_library(io.StringIO(yaml_content)) # --- valid cases --- @@ -421,7 +423,7 @@ def test_model_exposing_all_required_fields_is_valid() -> None: check_library_against_taxonomy(lib, taxonomy) # must not raise -# --- parse_yaml_library wiring --- +# --- validate_libraries_against_taxonomy --- _CONFORMING_LIB = """ library: @@ -458,34 +460,42 @@ def test_model_exposing_all_required_fields_is_valid() -> None: """ -def test_parse_library_declaring_taxonomy_is_checked() -> None: +def test_library_declaring_taxonomy_is_checked() -> None: taxonomy = _make_taxonomy(_make_category("production", ["injection_port"])) - lib = _parse_lib(_CONFORMING_LIB, taxonomy) + lib = _parse_lib(_CONFORMING_LIB) + validate_libraries_against_taxonomy([lib], taxonomy) # must not raise assert lib.taxonomy == "test_taxonomy" -def test_parse_library_declaring_taxonomy_raises_on_violation() -> None: +def test_library_declaring_taxonomy_raises_on_violation() -> None: taxonomy = _make_taxonomy(_make_category("production", ["injection_port"])) with pytest.raises(ValueError, match="injection_port"): - _parse_lib(_VIOLATING_LIB, taxonomy) + validate_libraries_against_taxonomy([_parse_lib(_VIOLATING_LIB)], taxonomy) -def test_parse_library_declaring_taxonomy_without_argument_raises() -> None: +def test_library_declaring_taxonomy_without_argument_raises() -> None: with pytest.raises(ValueError, match="no taxonomy was provided"): - _parse_lib(_CONFORMING_LIB) + validate_libraries_against_taxonomy([_parse_lib(_CONFORMING_LIB)], None) -def test_parse_library_with_mismatched_taxonomy_id_raises() -> None: +def test_library_with_mismatched_taxonomy_id_raises() -> None: taxonomy = Taxonomy( id="other_taxonomy", categories=[_make_category("production", ["injection_port"])], ) with pytest.raises(ValueError, match="other_taxonomy"): - _parse_lib(_CONFORMING_LIB, taxonomy) + validate_libraries_against_taxonomy([_parse_lib(_CONFORMING_LIB)], taxonomy) -def test_parse_library_without_declared_taxonomy_is_not_checked() -> None: +def test_library_without_declared_taxonomy_is_not_checked() -> None: """A model may carry a taxonomy-category without the library opting in.""" taxonomy = _make_taxonomy(_make_category("production", ["injection_port"])) - lib = _parse_lib(_LIB_WITHOUT_TAXONOMY, taxonomy) + lib = _parse_lib(_LIB_WITHOUT_TAXONOMY) + validate_libraries_against_taxonomy([lib], taxonomy) # must not raise assert lib.taxonomy is None + + +def test_parsing_a_declaring_library_does_not_validate_it() -> None: + """Parsing is pure reading: conformance is the caller's business.""" + lib = _parse_lib(_VIOLATING_LIB) # must not raise + assert lib.taxonomy == "test_taxonomy" diff --git a/tests/unittests/gems_craft/system_parsing/test_components_parsing.py b/tests/unittests/gems_craft/system_parsing/test_components_parsing.py index 7173a3fe..50606143 100644 --- a/tests/unittests/gems_craft/system_parsing/test_components_parsing.py +++ b/tests/unittests/gems_craft/system_parsing/test_components_parsing.py @@ -18,7 +18,8 @@ load_input_system, parse_yaml_system, ) -from gems_craft.study.resolve_components import consistency_check, resolve_system +from gems_craft.study.resolve_components import resolve_system +from gems_craft.study.validation import consistency_check COMPO_FILE = Path(__file__).parent / "systems/system.yml" From 90113714f3eba5962b5eb7280dc2ee1da9db1dea Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 10:52:32 +0000 Subject: [PATCH 2/2] refactor(study): disambiguate the two consistency checks `Study.check_consistency()` and `study/validation.py`'s `consistency_check()` were word-order permutations of each other, both under `gems_craft.study` and both raising ValueError about "consistency", while checking disjoint things. - `consistency_check` -> `check_component_models`: components reference a model id known to the library. - `Study.check_consistency()` -> `validation.check_data_requirements(study)`: the database covers every model parameter with a matching time/scenario structure. This resolves the TODO left by #265 and leaves `study.py` as a plain container. No behavior change; names and import paths only. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015p7onnyM3PYwpc7JhgKj1t --- docs/CHANGELOG.md | 9 ++++- src/gems_craft/study/folder.py | 4 +- src/gems_craft/study/study.py | 33 ++--------------- src/gems_craft/study/validation.py | 37 ++++++++++++++++--- src/gems_runner/simulation/optimization.py | 5 ++- tests/e2e/functional/perf_pypsa.py | 4 +- .../test_component_dependent_time_shift.py | 4 +- .../functional/test_libs_yaml_system_yaml.py | 6 +-- tests/e2e/functional/test_scenario_builder.py | 4 +- .../system/test_data_consistency.py | 17 +++++---- .../system_parsing/test_components_parsing.py | 10 ++--- 11 files changed, 69 insertions(+), 64 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6b8ed071..022be60c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,8 +18,13 @@ All notable changes to GemsPy are documented here. - **Breaking** - `check_library_against_taxonomy` moved from `gems_craft.model.taxonomy` to the new `gems_craft.model.validation`, and `consistency_check` from `gems_craft.study.resolve_components` to the new - `gems_craft.study.validation`. Reading and validating are now separate modules, - as in `optim_config/`. No behavior change; import paths only. + `gems_craft.study.validation`, where it is renamed + `check_component_models`. The `Study.check_consistency()` method moved to + that same module as the function `check_data_requirements(study)` — the two + checks are disjoint (component model ids vs. database coverage) and their + near-identical old names invited confusion. Reading and validating are now + separate modules, as in `optim_config/`. No behavior change; names and + import paths only. --- diff --git a/src/gems_craft/study/folder.py b/src/gems_craft/study/folder.py index 838b1bba..5ef9d306 100644 --- a/src/gems_craft/study/folder.py +++ b/src/gems_craft/study/folder.py @@ -23,7 +23,7 @@ ) from gems_craft.study.scenario_builder import ScenarioBuilder from gems_craft.study.study import Study -from gems_craft.study.validation import consistency_check +from gems_craft.study.validation import check_component_models def load_study(study_dir: Path) -> Study: @@ -61,7 +61,7 @@ def load_study(study_dir: Path) -> Study: model_dict: dict[str, Model] = {} for library in lib_dict.values(): model_dict |= library.models - consistency_check(system, model_dict) + check_component_models(system, model_dict) scenario_builder_path = ( study_dir / "input" / "data-series" / "modeler-scenariobuilder.dat" diff --git a/src/gems_craft/study/study.py b/src/gems_craft/study/study.py index 4c63db00..431ebd46 100644 --- a/src/gems_craft/study/study.py +++ b/src/gems_craft/study/study.py @@ -28,10 +28,9 @@ class Study: DataBase (parameter values for those components). These two objects are always used together to build an optimisation - problem. ``Study`` gathers them into a single, coherent unit and - provides the cross-validation logic that was previously spread between - ``DataBase.requirements_consistency`` and the callers of - ``build_problem``. + problem. ``Study`` gathers them into a single, coherent unit; the + cross-validation of the pair lives in ``study/validation.py`` + (``check_data_requirements``). """ system: System @@ -52,29 +51,3 @@ def models(self) -> Dict[str, Model]: return { mk: components[0].model for mk, components in self.model_components.items() } - - # TODO: this is a second, disjoint consistency check alongside - # study/validation.py's consistency_check() — consider consolidating both - # into that validation module. - def check_consistency(self) -> None: - """Validate that the database supplies data for every parameter of every - component defined in the system. - - Raises - ------ - ValueError - If a required data entry is missing or its time/scenario structure - does not match what the model parameter expects. - """ - for component in self.system.components: - for param in component.model.parameters.values(): - data_structure = self.database.get_data(component.id, param.name) - - if not data_structure.check_requirement( - component.model.parameters[param.name].structure.time, - component.model.parameters[param.name].structure.scenario, - ): - raise ValueError( - f"Data inconsistency for component: {component.id}, " - f"parameter: {param.name}. Requirement not met." - ) diff --git a/src/gems_craft/study/validation.py b/src/gems_craft/study/validation.py index 924a1486..1539c0f5 100644 --- a/src/gems_craft/study/validation.py +++ b/src/gems_craft/study/validation.py @@ -10,25 +10,26 @@ # # This file is part of the Antares project. -"""Cross-validation of a resolved system against the models it refers to. +"""Cross-validation of a resolved system against the models and data it refers to. -Kept apart from `resolve_components.py`, which only resolves the parsed system -into its runtime objects — mirroring `optim_config/parsing.py` and -`optim_config/validation.py`. +Kept apart from `resolve_components.py` (which only resolves the parsed system +into its runtime objects) and from `study.py` (which only holds them together) +— mirroring `optim_config/parsing.py` and `optim_config/validation.py`. """ from typing import Dict from gems_craft.model import Model +from gems_craft.study.study import Study from gems_craft.study.system import System -def consistency_check(system: System, input_models: Dict[str, Model]) -> bool: +def check_component_models(system: System, input_models: Dict[str, Model]) -> bool: """ Checks if all components in the System have a valid model from the library. Returns True if all components are consistent, raises ValueError otherwise. """ - # TODO: Update this consistency check to check if each component have a valid model from the lib it refers to (and not all libs) + # TODO: Update this check to verify that each component has a valid model from the lib it refers to (and not all libs) model_ids_set = input_models.keys() for component in system.all_components: if component.model.id not in model_ids_set: @@ -36,3 +37,27 @@ def consistency_check(system: System, input_models: Dict[str, Model]) -> bool: f"Error: Component {component.id} has invalid model ID: {component.model.id}" ) return True + + +def check_data_requirements(study: Study) -> None: + """Validate that the database supplies data for every parameter of every + component defined in the system. + + Raises + ------ + ValueError + If a required data entry is missing or its time/scenario structure + does not match what the model parameter expects. + """ + for component in study.system.components: + for param in component.model.parameters.values(): + data_structure = study.database.get_data(component.id, param.name) + + if not data_structure.check_requirement( + component.model.parameters[param.name].structure.time, + component.model.parameters[param.name].structure.scenario, + ): + raise ValueError( + f"Data inconsistency for component: {component.id}, " + f"parameter: {param.name}. Requirement not met." + ) diff --git a/src/gems_runner/simulation/optimization.py b/src/gems_runner/simulation/optimization.py index 71752c58..30706219 100644 --- a/src/gems_runner/simulation/optimization.py +++ b/src/gems_runner/simulation/optimization.py @@ -48,6 +48,7 @@ from gems_craft.study.parsing import IntegerStrategyId from gems_craft.study.study import Study from gems_craft.study.system import Component +from gems_craft.study.validation import check_data_requirements from gems_runner.simulation.linearize import ( VectorizedExpr, VectorizedLinearExprBuilder, @@ -1071,7 +1072,7 @@ def build_problem( problem is built. Entries whose variable is time-independent, or is absent from this block, are ignored. """ - study.check_consistency() + check_data_requirements(study) oob_filter = OutOfBoundsFilter(optim_config) if optim_config is not None else None builder = _OptimizationProblemBuilder( @@ -1145,7 +1146,7 @@ def build_decomposed_problems( """ from gems_craft.optim_config.parsing import ElementLocation - study.check_consistency() + check_data_requirements(study) oob_filter = OutOfBoundsFilter(optim_config) diff --git a/tests/e2e/functional/perf_pypsa.py b/tests/e2e/functional/perf_pypsa.py index 6fd91fe3..e5f26433 100644 --- a/tests/e2e/functional/perf_pypsa.py +++ b/tests/e2e/functional/perf_pypsa.py @@ -15,7 +15,7 @@ resolve_system, ) from gems_craft.study.system import System -from gems_craft.study.validation import consistency_check +from gems_craft.study.validation import check_component_models from gems_runner.simulation import TimeBlock, build_problem @@ -30,7 +30,7 @@ def setup_data(pypsa_dir: Path) -> Tuple[System, DataBase]: input_study = parse_yaml_system(c) lib_dict = resolve_library([input_library]) system = resolve_system(input_study, lib_dict) - consistency_check(system, lib_dict["pypsa_models"].models) + check_component_models(system, lib_dict["pypsa_models"].models) database = build_data_base(input_study, series_dir) return system, database diff --git a/tests/e2e/functional/test_component_dependent_time_shift.py b/tests/e2e/functional/test_component_dependent_time_shift.py index 070976c5..983e1115 100644 --- a/tests/e2e/functional/test_component_dependent_time_shift.py +++ b/tests/e2e/functional/test_component_dependent_time_shift.py @@ -102,7 +102,7 @@ build_data_base, resolve_system, ) -from gems_craft.study.validation import consistency_check +from gems_craft.study.validation import check_component_models from gems_runner.simulation import TimeBlock, build_problem from tests.e2e.functional.libs.standard import ( BALANCE_PORT_TYPE, @@ -413,7 +413,7 @@ def test_two_components_different_lags_yaml( lib_dict = resolve_library([input_library]) system = resolve_system(input_system, lib_dict) - consistency_check(system, lib_dict["time_shift_test"].models) + check_component_models(system, lib_dict["time_shift_test"].models) database = build_data_base(input_system, timeseries_dir=_series_dir) diff --git a/tests/e2e/functional/test_libs_yaml_system_yaml.py b/tests/e2e/functional/test_libs_yaml_system_yaml.py index c8d3bad9..99c130d3 100644 --- a/tests/e2e/functional/test_libs_yaml_system_yaml.py +++ b/tests/e2e/functional/test_libs_yaml_system_yaml.py @@ -51,7 +51,7 @@ ) from gems_craft.study.study import Study from gems_craft.study.system import System -from gems_craft.study.validation import consistency_check +from gems_craft.study.validation import check_component_models from gems_runner.simulation import TimeBlock, build_problem @@ -60,7 +60,7 @@ def test_basic_balance_using_yaml( ) -> None: result_lib = resolve_library([input_library]) system = resolve_system(input_system, result_lib) - consistency_check(system, result_lib["basic"].models) + check_component_models(system, result_lib["basic"].models) database = build_data_base(input_system, None) @@ -87,7 +87,7 @@ def _setup_test(study_file_name: str) -> Study: input_system = parse_yaml_system(c) lib_dict = resolve_library([input_library]) system = resolve_system(input_system, lib_dict) - consistency_check(system, lib_dict["basic"].models) + check_component_models(system, lib_dict["basic"].models) database = build_data_base(input_system, series_dir) return Study(system, database) diff --git a/tests/e2e/functional/test_scenario_builder.py b/tests/e2e/functional/test_scenario_builder.py index 7a20dc1e..f46f884f 100644 --- a/tests/e2e/functional/test_scenario_builder.py +++ b/tests/e2e/functional/test_scenario_builder.py @@ -24,7 +24,7 @@ resolve_system, ) from gems_craft.study.scenario_builder import ScenarioBuilder -from gems_craft.study.validation import consistency_check +from gems_craft.study.validation import check_component_models from gems_runner.simulation import build_problem from gems_runner.simulation.time_block import TimeBlock @@ -58,7 +58,7 @@ def test_system_with_scenarization( yaml_comp = parse_yaml_system(file) components = resolve_system(yaml_comp, lib_dict) - consistency_check(components, lib_dict["basic"].models) + check_component_models(components, lib_dict["basic"].models) timeblock = TimeBlock(1, list(range(2))) problem = build_problem(Study(components, database), timeblock, list(range(3))) diff --git a/tests/unittests/gems_craft/system/test_data_consistency.py b/tests/unittests/gems_craft/system/test_data_consistency.py index 400a1990..dcab18b1 100644 --- a/tests/unittests/gems_craft/system/test_data_consistency.py +++ b/tests/unittests/gems_craft/system/test_data_consistency.py @@ -41,6 +41,7 @@ create_component, ) from gems_craft.study.data import load_ts_from_file +from gems_craft.study.validation import check_data_requirements from tests.unittests.gems_craft.system.libs.standard import ( BALANCE_PORT_TYPE, CONSTANT, @@ -152,7 +153,7 @@ def test_requirements_consistency_demand_model_fix_ok( # When # No ValueError should be raised - Study(mock_network, database).check_consistency() + check_data_requirements(Study(mock_network, database)) def test_requirements_consistency_generator_model_ok(mock_network: System) -> None: @@ -165,7 +166,7 @@ def test_requirements_consistency_generator_model_ok(mock_network: System) -> No database.add_data("D", "demand", ConstantData(30)) # When - Study(mock_network, database).check_consistency() + check_data_requirements(Study(mock_network, database)) def test_consistency_generation_time_free_for_constant_model_raises_exception( @@ -184,7 +185,7 @@ def test_consistency_generation_time_free_for_constant_model_raises_exception( # When with pytest.raises(ValueError, match="Data inconsistency"): - Study(mock_network, database).check_consistency() + check_data_requirements(Study(mock_network, database)) def test_requirements_consistency_demand_model_time_varying_ok( @@ -200,7 +201,7 @@ def test_requirements_consistency_demand_model_time_varying_ok( # When # No ValueError should be raised - Study(mock_network, database).check_consistency() + check_data_requirements(Study(mock_network, database)) def test_requirements_consistency_time_varying_parameter_with_correct_data_passes( @@ -225,7 +226,7 @@ def test_requirements_consistency_time_varying_parameter_with_correct_data_passe system.connect(PortRef(gen, "balance_port"), PortRef(node, "balance_port")) # No ValueError should be raised - Study(system, database).check_consistency() + check_data_requirements(Study(system, database)) @pytest.mark.parametrize( @@ -270,7 +271,7 @@ def test_requirements_consistency_time_varying_parameter_with_scenario_varying_d # When # ValueError should be raised with pytest.raises(ValueError, match="Data inconsistency"): - Study(system, database).check_consistency() + check_data_requirements(Study(system, database)) @pytest.mark.parametrize( @@ -306,7 +307,7 @@ def test_requirements_consistency_scenario_varying_parameter_with_time_varying_d # ValueError should be raised with pytest.raises(ValueError, match="Data inconsistency"): - Study(system, database).check_consistency() + check_data_requirements(Study(system, database)) def test_requirements_consistency_scenario_varying_parameter_with_correct_data_passes( @@ -331,7 +332,7 @@ def test_requirements_consistency_scenario_varying_parameter_with_correct_data_p system.add_component(gen) # No ValueError should be raised - Study(system, database).check_consistency() + check_data_requirements(Study(system, database)) def test_load_data_from_txt() -> None: diff --git a/tests/unittests/gems_craft/system_parsing/test_components_parsing.py b/tests/unittests/gems_craft/system_parsing/test_components_parsing.py index 50606143..00a3978d 100644 --- a/tests/unittests/gems_craft/system_parsing/test_components_parsing.py +++ b/tests/unittests/gems_craft/system_parsing/test_components_parsing.py @@ -19,7 +19,7 @@ parse_yaml_system, ) from gems_craft.study.resolve_components import resolve_system -from gems_craft.study.validation import consistency_check +from gems_craft.study.validation import check_component_models COMPO_FILE = Path(__file__).parent / "systems/system.yml" @@ -51,12 +51,12 @@ def test_parsing_components_ok( assert len(result.connections) == 2 -def test_consistency_check_ok( +def test_check_component_models_ok( input_system: SystemSchema, input_library: LibrarySchema ) -> None: result_lib = resolve_library([input_library]) result_system = resolve_system(input_system, result_lib) - consistency_check(result_system, result_lib["basic"].models) + check_component_models(result_system, result_lib["basic"].models) def test_load_input_system_ok(tmp_path: Path) -> None: @@ -94,7 +94,7 @@ def test_load_input_system_missing_file_raises_error() -> None: load_input_system(missing) -def test_consistency_check_ko( +def test_check_component_models_ko( input_system: SystemSchema, input_library: LibrarySchema ) -> None: result_lib = resolve_library([input_library]) @@ -104,7 +104,7 @@ def test_consistency_check_ko( ValueError, match=r"Error: Component G has invalid model ID: basic.generator", ): - consistency_check(result_comp, result_lib["basic"].models) + check_component_models(result_comp, result_lib["basic"].models) # ---------------------------------------------------------------------------